1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
11 // On AArch64, stack frames are structured as follows:
13 // The stack grows downward.
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
19 // At function entry, the "frame" looks as follows:
22 // |-----------------------------------|
24 // | arguments passed on the stack |
26 // |-----------------------------------| <- sp
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
37 // |-----------------------------------|
39 // | arguments passed on the stack |
41 // |-----------------------------------|
43 // | (Win64 only) varargs from reg |
45 // |-----------------------------------|
47 // | callee-saved gpr registers | <--.
48 // | | | On Darwin platforms these
49 // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50 // | | | (frame record first)
51 // | prev_fp, prev_lr | <--'
52 // | (a.k.a. "frame record") |
53 // |-----------------------------------| <- fp(=x29)
55 // | callee-saved fp/simd/SVE regs |
57 // |-----------------------------------|
59 // | SVE stack objects |
61 // |-----------------------------------|
62 // |.empty.space.to.make.part.below....|
63 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
64 // |.the.standard.16-byte.alignment....| compile time; if present)
65 // |-----------------------------------|
67 // | local variables of fixed size |
68 // | including spill slots |
69 // |-----------------------------------| <- bp(not defined by ABI,
70 // |.variable-sized.local.variables....| LLVM chooses X19)
71 // |.(VLAs)............................| (size of this area is unknown at
72 // |...................................| compile time)
73 // |-----------------------------------| <- sp
77 // To access the data in a frame, at-compile time, a constant offset must be
78 // computable from one of the pointers (fp, bp, sp) to access it. The size
79 // of the areas with a dotted background cannot be computed at compile-time
80 // if they are present, making it required to have all three of fp, bp and
81 // sp to be set up to be able to access all contents in the frame areas,
82 // assuming all of the frame areas are non-empty.
84 // For most functions, some of the frame areas are empty. For those functions,
85 // it may not be necessary to set up fp or bp:
86 // * A base pointer is definitely needed when there are both VLAs and local
87 // variables with more-than-default alignment requirements.
88 // * A frame pointer is definitely needed when there are local variables with
89 // more-than-default alignment requirements.
91 // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
92 // callee-saved area, since the unwind encoding does not allow for encoding
93 // this dynamically and existing tools depend on this layout. For other
94 // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
95 // area to allow SVE stack objects (allocated directly below the callee-saves,
96 // if available) to be accessed directly from the framepointer.
97 // The SVE spill/fill instructions have VL-scaled addressing modes such
99 // ldr z8, [fp, #-7 mul vl]
100 // For SVE the size of the vector length (VL) is not known at compile-time, so
101 // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
102 // layout, we don't need to add an unscaled offset to the framepointer before
103 // accessing the SVE object in the frame.
105 // In some cases when a base pointer is not strictly needed, it is generated
106 // anyway when offsets from the frame pointer to access local variables become
107 // so large that the offset can't be encoded in the immediate fields of loads
110 // FIXME: also explain the redzone concept.
111 // FIXME: also explain the concept of reserved call frames.
113 //===----------------------------------------------------------------------===//
115 #include "AArch64FrameLowering.h"
116 #include "AArch64InstrInfo.h"
117 #include "AArch64MachineFunctionInfo.h"
118 #include "AArch64RegisterInfo.h"
119 #include "AArch64StackOffset.h"
120 #include "AArch64Subtarget.h"
121 #include "AArch64TargetMachine.h"
122 #include "MCTargetDesc/AArch64AddressingModes.h"
123 #include "llvm/ADT/ScopeExit.h"
124 #include "llvm/ADT/SmallVector.h"
125 #include "llvm/ADT/Statistic.h"
126 #include "llvm/CodeGen/LivePhysRegs.h"
127 #include "llvm/CodeGen/MachineBasicBlock.h"
128 #include "llvm/CodeGen/MachineFrameInfo.h"
129 #include "llvm/CodeGen/MachineFunction.h"
130 #include "llvm/CodeGen/MachineInstr.h"
131 #include "llvm/CodeGen/MachineInstrBuilder.h"
132 #include "llvm/CodeGen/MachineMemOperand.h"
133 #include "llvm/CodeGen/MachineModuleInfo.h"
134 #include "llvm/CodeGen/MachineOperand.h"
135 #include "llvm/CodeGen/MachineRegisterInfo.h"
136 #include "llvm/CodeGen/RegisterScavenging.h"
137 #include "llvm/CodeGen/TargetInstrInfo.h"
138 #include "llvm/CodeGen/TargetRegisterInfo.h"
139 #include "llvm/CodeGen/TargetSubtargetInfo.h"
140 #include "llvm/CodeGen/WinEHFuncInfo.h"
141 #include "llvm/IR/Attributes.h"
142 #include "llvm/IR/CallingConv.h"
143 #include "llvm/IR/DataLayout.h"
144 #include "llvm/IR/DebugLoc.h"
145 #include "llvm/IR/Function.h"
146 #include "llvm/MC/MCAsmInfo.h"
147 #include "llvm/MC/MCDwarf.h"
148 #include "llvm/Support/CommandLine.h"
149 #include "llvm/Support/Debug.h"
150 #include "llvm/Support/ErrorHandling.h"
151 #include "llvm/Support/MathExtras.h"
152 #include "llvm/Support/raw_ostream.h"
153 #include "llvm/Target/TargetMachine.h"
154 #include "llvm/Target/TargetOptions.h"
160 using namespace llvm;
162 #define DEBUG_TYPE "frame-info"
164 static cl::opt<bool> EnableRedZone("aarch64-redzone",
165 cl::desc("enable use of redzone on AArch64"),
166 cl::init(false), cl::Hidden);
169 ReverseCSRRestoreSeq("reverse-csr-restore-seq",
170 cl::desc("reverse the CSR restore sequence"),
171 cl::init(false), cl::Hidden);
173 static cl::opt<bool> StackTaggingMergeSetTag(
174 "stack-tagging-merge-settag",
175 cl::desc("merge settag instruction in function epilog"), cl::init(true),
178 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
180 /// Returns the argument pop size.
181 static uint64_t getArgumentPopSize(MachineFunction &MF,
182 MachineBasicBlock &MBB) {
183 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
184 bool IsTailCallReturn = false;
185 if (MBB.end() != MBBI) {
186 unsigned RetOpcode = MBBI->getOpcode();
187 IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
188 RetOpcode == AArch64::TCRETURNri ||
189 RetOpcode == AArch64::TCRETURNriBTI;
191 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
193 uint64_t ArgumentPopSize = 0;
194 if (IsTailCallReturn) {
195 MachineOperand &StackAdjust = MBBI->getOperand(1);
197 // For a tail-call in a callee-pops-arguments environment, some or all of
198 // the stack may actually be in use for the call's arguments, this is
199 // calculated during LowerCall and consumed here...
200 ArgumentPopSize = StackAdjust.getImm();
202 // ... otherwise the amount to pop is *all* of the argument space,
203 // conveniently stored in the MachineFunctionInfo by
204 // LowerFormalArguments. This will, of course, be zero for the C calling
206 ArgumentPopSize = AFI->getArgumentStackToRestore();
209 return ArgumentPopSize;
212 /// This is the biggest offset to the stack pointer we can encode in aarch64
213 /// instructions (without using a separate calculation and a temp register).
214 /// Note that the exception here are vector stores/loads which cannot encode any
215 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
216 static const unsigned DefaultSafeSPDisplacement = 255;
218 /// Look at each instruction that references stack frames and return the stack
219 /// size limit beyond which some of these instructions will require a scratch
220 /// register during their expansion later.
221 static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
222 // FIXME: For now, just conservatively guestimate based on unscaled indexing
223 // range. We'll end up allocating an unnecessary spill slot a lot, but
224 // realistically that's not a big deal at this stage of the game.
225 for (MachineBasicBlock &MBB : MF) {
226 for (MachineInstr &MI : MBB) {
227 if (MI.isDebugInstr() || MI.isPseudo() ||
228 MI.getOpcode() == AArch64::ADDXri ||
229 MI.getOpcode() == AArch64::ADDSXri)
232 for (const MachineOperand &MO : MI.operands()) {
237 if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
238 AArch64FrameOffsetCannotUpdate)
243 return DefaultSafeSPDisplacement;
247 AArch64FrameLowering::getStackIDForScalableVectors() const {
248 return TargetStackID::SVEVector;
251 /// Returns the size of the fixed object area (allocated next to sp on entry)
252 /// On Win64 this may include a var args area and an UnwindHelp object for EH.
253 static unsigned getFixedObjectSize(const MachineFunction &MF,
254 const AArch64FunctionInfo *AFI, bool IsWin64,
256 if (!IsWin64 || IsFunclet) {
257 // Only Win64 uses fixed objects, and then only for the function (not
261 // Var args are stored here in the primary function.
262 const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
263 // To support EH funclets we allocate an UnwindHelp object
264 const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
265 return alignTo(VarArgsArea + UnwindHelpObject, 16);
269 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
270 static StackOffset getSVEStackSize(const MachineFunction &MF) {
271 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
272 return {(int64_t)AFI->getStackSizeSVE(), MVT::nxv1i8};
275 bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
278 // Don't use the red zone if the function explicitly asks us not to.
279 // This is typically used for kernel code.
280 if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
283 const MachineFrameInfo &MFI = MF.getFrameInfo();
284 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
285 uint64_t NumBytes = AFI->getLocalStackSize();
287 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128 ||
288 getSVEStackSize(MF));
291 /// hasFP - Return true if the specified function should have a dedicated frame
292 /// pointer register.
293 bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const {
294 const MachineFrameInfo &MFI = MF.getFrameInfo();
295 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
296 // Win64 EH requires a frame pointer if funclets are present, as the locals
297 // are accessed off the frame pointer in both the parent function and the
299 if (MF.hasEHFunclets())
301 // Retain behavior of always omitting the FP for leaf functions when possible.
302 if (MF.getTarget().Options.DisableFramePointerElim(MF))
304 if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
305 MFI.hasStackMap() || MFI.hasPatchPoint() ||
306 RegInfo->needsStackRealignment(MF))
308 // With large callframes around we may need to use FP to access the scavenging
309 // emergency spillslot.
311 // Unfortunately some calls to hasFP() like machine verifier ->
312 // getReservedReg() -> hasFP in the middle of global isel are too early
313 // to know the max call frame size. Hopefully conservatively returning "true"
314 // in those cases is fine.
315 // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
316 if (!MFI.isMaxCallFrameSizeComputed() ||
317 MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
323 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
324 /// not required, we reserve argument space for call sites in the function
325 /// immediately on entry to the current function. This eliminates the need for
326 /// add/sub sp brackets around call sites. Returns true if the call frame is
327 /// included as part of the stack frame.
329 AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
330 return !MF.getFrameInfo().hasVarSizedObjects();
333 MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
334 MachineFunction &MF, MachineBasicBlock &MBB,
335 MachineBasicBlock::iterator I) const {
336 const AArch64InstrInfo *TII =
337 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
338 DebugLoc DL = I->getDebugLoc();
339 unsigned Opc = I->getOpcode();
340 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
341 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
343 if (!hasReservedCallFrame(MF)) {
344 int64_t Amount = I->getOperand(0).getImm();
345 Amount = alignTo(Amount, getStackAlign());
349 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
350 // doesn't have to pop anything), then the first operand will be zero too so
351 // this adjustment is a no-op.
352 if (CalleePopAmount == 0) {
353 // FIXME: in-function stack adjustment for calls is limited to 24-bits
354 // because there's no guaranteed temporary register available.
356 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
357 // 1) For offset <= 12-bit, we use LSL #0
358 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
359 // LSL #0, and the other uses LSL #12.
361 // Most call frames will be allocated at the start of a function so
362 // this is OK, but it is a limitation that needs dealing with.
363 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
364 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, {Amount, MVT::i8},
367 } else if (CalleePopAmount != 0) {
368 // If the calling convention demands that the callee pops arguments from the
369 // stack, we want to add it back if we have a reserved call frame.
370 assert(CalleePopAmount < 0xffffff && "call frame too large");
371 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
372 {-(int64_t)CalleePopAmount, MVT::i8}, TII);
377 static bool ShouldSignReturnAddress(MachineFunction &MF) {
378 // The function should be signed in the following situations:
379 // - sign-return-address=all
380 // - sign-return-address=non-leaf and the functions spills the LR
382 const Function &F = MF.getFunction();
383 if (!F.hasFnAttribute("sign-return-address"))
386 StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
387 if (Scope.equals("none"))
390 if (Scope.equals("all"))
393 assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
395 for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
396 if (Info.getReg() == AArch64::LR)
402 void AArch64FrameLowering::emitCalleeSavedFrameMoves(
403 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
404 MachineFunction &MF = *MBB.getParent();
405 MachineFrameInfo &MFI = MF.getFrameInfo();
406 const TargetSubtargetInfo &STI = MF.getSubtarget();
407 const MCRegisterInfo *MRI = STI.getRegisterInfo();
408 const TargetInstrInfo *TII = STI.getInstrInfo();
409 DebugLoc DL = MBB.findDebugLoc(MBBI);
411 // Add callee saved registers to move list.
412 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
416 for (const auto &Info : CSI) {
417 unsigned Reg = Info.getReg();
419 MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
420 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
421 unsigned CFIIndex = MF.addFrameInst(
422 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
423 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
424 .addCFIIndex(CFIIndex)
425 .setMIFlags(MachineInstr::FrameSetup);
429 // Find a scratch register that we can use at the start of the prologue to
430 // re-align the stack pointer. We avoid using callee-save registers since they
431 // may appear to be free when this is called from canUseAsPrologue (during
432 // shrink wrapping), but then no longer be free when this is called from
435 // FIXME: This is a bit conservative, since in the above case we could use one
436 // of the callee-save registers as a scratch temp to re-align the stack pointer,
437 // but we would then have to make sure that we were in fact saving at least one
438 // callee-save register in the prologue, which is additional complexity that
439 // doesn't seem worth the benefit.
440 static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB) {
441 MachineFunction *MF = MBB->getParent();
443 // If MBB is an entry block, use X9 as the scratch register
444 if (&MF->front() == MBB)
447 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
448 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
449 LivePhysRegs LiveRegs(TRI);
450 LiveRegs.addLiveIns(*MBB);
452 // Mark callee saved registers as used so we will not choose them.
453 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
454 for (unsigned i = 0; CSRegs[i]; ++i)
455 LiveRegs.addReg(CSRegs[i]);
457 // Prefer X9 since it was historically used for the prologue scratch reg.
458 const MachineRegisterInfo &MRI = MF->getRegInfo();
459 if (LiveRegs.available(MRI, AArch64::X9))
462 for (unsigned Reg : AArch64::GPR64RegClass) {
463 if (LiveRegs.available(MRI, Reg))
466 return AArch64::NoRegister;
469 bool AArch64FrameLowering::canUseAsPrologue(
470 const MachineBasicBlock &MBB) const {
471 const MachineFunction *MF = MBB.getParent();
472 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
473 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
474 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
476 // Don't need a scratch register if we're not going to re-align the stack.
477 if (!RegInfo->needsStackRealignment(*MF))
479 // Otherwise, we can use any block as long as it has a scratch register
481 return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
484 static bool windowsRequiresStackProbe(MachineFunction &MF,
485 uint64_t StackSizeInBytes) {
486 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
487 if (!Subtarget.isTargetWindows())
489 const Function &F = MF.getFunction();
490 // TODO: When implementing stack protectors, take that into account
491 // for the probe threshold.
492 unsigned StackProbeSize = 4096;
493 if (F.hasFnAttribute("stack-probe-size"))
494 F.getFnAttribute("stack-probe-size")
496 .getAsInteger(0, StackProbeSize);
497 return (StackSizeInBytes >= StackProbeSize) &&
498 !F.hasFnAttribute("no-stack-arg-probe");
501 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
502 MachineFunction &MF, uint64_t StackBumpBytes) const {
503 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
504 const MachineFrameInfo &MFI = MF.getFrameInfo();
505 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
506 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
508 if (AFI->getLocalStackSize() == 0)
511 // 512 is the maximum immediate for stp/ldp that will be used for
512 // callee-save save/restores
513 if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
516 if (MFI.hasVarSizedObjects())
519 if (RegInfo->needsStackRealignment(MF))
522 // This isn't strictly necessary, but it simplifies things a bit since the
523 // current RedZone handling code assumes the SP is adjusted by the
524 // callee-save save/restore code.
525 if (canUseRedZone(MF))
528 // When there is an SVE area on the stack, always allocate the
529 // callee-saves and spills/locals separately.
530 if (getSVEStackSize(MF))
536 bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
537 MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
538 if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
544 // Disable combined SP bump if the last instruction is an MTE tag store. It
545 // is almost always better to merge SP adjustment into those instructions.
546 MachineBasicBlock::iterator LastI = MBB.getFirstTerminator();
547 MachineBasicBlock::iterator Begin = MBB.begin();
548 while (LastI != Begin) {
550 if (LastI->isTransient())
552 if (!LastI->getFlag(MachineInstr::FrameDestroy))
555 switch (LastI->getOpcode()) {
556 case AArch64::STGloop:
557 case AArch64::STZGloop:
558 case AArch64::STGOffset:
559 case AArch64::STZGOffset:
560 case AArch64::ST2GOffset:
561 case AArch64::STZ2GOffset:
566 llvm_unreachable("unreachable");
569 // Given a load or a store instruction, generate an appropriate unwinding SEH
571 static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
572 const TargetInstrInfo &TII,
573 MachineInstr::MIFlag Flag) {
574 unsigned Opc = MBBI->getOpcode();
575 MachineBasicBlock *MBB = MBBI->getParent();
576 MachineFunction &MF = *MBB->getParent();
577 DebugLoc DL = MBBI->getDebugLoc();
578 unsigned ImmIdx = MBBI->getNumOperands() - 1;
579 int Imm = MBBI->getOperand(ImmIdx).getImm();
580 MachineInstrBuilder MIB;
581 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
582 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
586 llvm_unreachable("No SEH Opcode for this instruction");
587 case AArch64::LDPDpost:
590 case AArch64::STPDpre: {
591 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
592 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
593 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
600 case AArch64::LDPXpost:
603 case AArch64::STPXpre: {
604 Register Reg0 = MBBI->getOperand(1).getReg();
605 Register Reg1 = MBBI->getOperand(2).getReg();
606 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
607 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
611 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
612 .addImm(RegInfo->getSEHRegNum(Reg0))
613 .addImm(RegInfo->getSEHRegNum(Reg1))
618 case AArch64::LDRDpost:
621 case AArch64::STRDpre: {
622 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
623 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
629 case AArch64::LDRXpost:
632 case AArch64::STRXpre: {
633 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
634 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
641 case AArch64::LDPDi: {
642 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
643 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
644 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
652 case AArch64::LDPXi: {
653 Register Reg0 = MBBI->getOperand(0).getReg();
654 Register Reg1 = MBBI->getOperand(1).getReg();
655 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
656 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
660 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
661 .addImm(RegInfo->getSEHRegNum(Reg0))
662 .addImm(RegInfo->getSEHRegNum(Reg1))
667 case AArch64::STRXui:
668 case AArch64::LDRXui: {
669 int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
670 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
676 case AArch64::STRDui:
677 case AArch64::LDRDui: {
678 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
679 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
686 auto I = MBB->insertAfter(MBBI, MIB);
690 // Fix up the SEH opcode associated with the save/restore instruction.
691 static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
692 unsigned LocalStackSize) {
693 MachineOperand *ImmOpnd = nullptr;
694 unsigned ImmIdx = MBBI->getNumOperands() - 1;
695 switch (MBBI->getOpcode()) {
697 llvm_unreachable("Fix the offset in the SEH instruction");
698 case AArch64::SEH_SaveFPLR:
699 case AArch64::SEH_SaveRegP:
700 case AArch64::SEH_SaveReg:
701 case AArch64::SEH_SaveFRegP:
702 case AArch64::SEH_SaveFReg:
703 ImmOpnd = &MBBI->getOperand(ImmIdx);
707 ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
710 // Convert callee-save register save/restore instruction to do stack pointer
711 // decrement/increment to allocate/deallocate the callee-save stack area by
712 // converting store/load to use pre/post increment version.
713 static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
714 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
715 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
716 bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
717 // Ignore instructions that do not operate on SP, i.e. shadow call stack
718 // instructions and associated CFI instruction.
719 while (MBBI->getOpcode() == AArch64::STRXpost ||
720 MBBI->getOpcode() == AArch64::LDRXpre ||
721 MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
722 if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
723 assert(MBBI->getOperand(0).getReg() != AArch64::SP);
728 switch (MBBI->getOpcode()) {
730 llvm_unreachable("Unexpected callee-save save/restore opcode!");
732 NewOpc = AArch64::STPXpre;
736 NewOpc = AArch64::STPDpre;
740 NewOpc = AArch64::STPQpre;
743 case AArch64::STRXui:
744 NewOpc = AArch64::STRXpre;
746 case AArch64::STRDui:
747 NewOpc = AArch64::STRDpre;
749 case AArch64::STRQui:
750 NewOpc = AArch64::STRQpre;
753 NewOpc = AArch64::LDPXpost;
757 NewOpc = AArch64::LDPDpost;
761 NewOpc = AArch64::LDPQpost;
764 case AArch64::LDRXui:
765 NewOpc = AArch64::LDRXpost;
767 case AArch64::LDRDui:
768 NewOpc = AArch64::LDRDpost;
770 case AArch64::LDRQui:
771 NewOpc = AArch64::LDRQpost;
774 // Get rid of the SEH code associated with the old instruction.
776 auto SEH = std::next(MBBI);
777 if (AArch64InstrInfo::isSEHInstruction(*SEH))
778 SEH->eraseFromParent();
781 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
782 MIB.addReg(AArch64::SP, RegState::Define);
784 // Copy all operands other than the immediate offset.
785 unsigned OpndIdx = 0;
786 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
788 MIB.add(MBBI->getOperand(OpndIdx));
790 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
791 "Unexpected immediate offset in first/last callee-save save/restore "
793 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
794 "Unexpected base register in callee-save save/restore instruction!");
795 assert(CSStackSizeInc % Scale == 0);
796 MIB.addImm(CSStackSizeInc / Scale);
798 MIB.setMIFlags(MBBI->getFlags());
799 MIB.setMemRefs(MBBI->memoperands());
801 // Generate a new SEH code that corresponds to the new instruction.
804 InsertSEH(*MIB, *TII,
805 InProlog ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy);
808 return std::prev(MBB.erase(MBBI));
811 // Fixup callee-save register save/restore instructions to take into account
812 // combined SP bump by adding the local stack size to the stack offsets.
813 static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
814 uint64_t LocalStackSize,
817 if (AArch64InstrInfo::isSEHInstruction(MI))
820 unsigned Opc = MI.getOpcode();
822 // Ignore instructions that do not operate on SP, i.e. shadow call stack
823 // instructions and associated CFI instruction.
824 if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
825 Opc == AArch64::CFI_INSTRUCTION) {
826 if (Opc != AArch64::CFI_INSTRUCTION)
827 assert(MI.getOperand(0).getReg() != AArch64::SP);
834 case AArch64::STRXui:
836 case AArch64::STRDui:
838 case AArch64::LDRXui:
840 case AArch64::LDRDui:
844 case AArch64::STRQui:
846 case AArch64::LDRQui:
850 llvm_unreachable("Unexpected callee-save save/restore opcode!");
853 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
854 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
855 "Unexpected base register in callee-save save/restore instruction!");
856 // Last operand is immediate offset that needs fixing.
857 MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
858 // All generated opcodes have scaled offsets.
859 assert(LocalStackSize % Scale == 0);
860 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
864 auto MBBI = std::next(MachineBasicBlock::iterator(MI));
865 assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
866 assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
867 "Expecting a SEH instruction");
868 fixupSEHOpcode(MBBI, LocalStackSize);
872 static void adaptForLdStOpt(MachineBasicBlock &MBB,
873 MachineBasicBlock::iterator FirstSPPopI,
874 MachineBasicBlock::iterator LastPopI) {
875 // Sometimes (when we restore in the same order as we save), we can end up
876 // with code like this:
878 // ldp x26, x25, [sp]
879 // ldp x24, x23, [sp, #16]
880 // ldp x22, x21, [sp, #32]
881 // ldp x20, x19, [sp, #48]
884 // In this case, it is always better to put the first ldp at the end, so
885 // that the load-store optimizer can run and merge the ldp and the add into
887 // If we managed to grab the first pop instruction, move it to the end.
888 if (ReverseCSRRestoreSeq)
889 MBB.splice(FirstSPPopI, &MBB, LastPopI);
890 // We should end up with something like this now:
892 // ldp x24, x23, [sp, #16]
893 // ldp x22, x21, [sp, #32]
894 // ldp x20, x19, [sp, #48]
895 // ldp x26, x25, [sp]
898 // and the load-store optimizer can merge the last two instructions into:
900 // ldp x26, x25, [sp], #64
904 static bool ShouldSignWithAKey(MachineFunction &MF) {
905 const Function &F = MF.getFunction();
906 if (!F.hasFnAttribute("sign-return-address-key"))
909 const StringRef Key =
910 F.getFnAttribute("sign-return-address-key").getValueAsString();
911 assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
912 return Key.equals_lower("a_key");
915 static bool needsWinCFI(const MachineFunction &MF) {
916 const Function &F = MF.getFunction();
917 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
918 F.needsUnwindTableEntry();
921 static bool isTargetDarwin(const MachineFunction &MF) {
922 return MF.getSubtarget<AArch64Subtarget>().isTargetDarwin();
925 static bool isTargetWindows(const MachineFunction &MF) {
926 return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
929 // Convenience function to determine whether I is an SVE callee save.
930 static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
931 switch (I->getOpcode()) {
934 case AArch64::STR_ZXI:
935 case AArch64::STR_PXI:
936 case AArch64::LDR_ZXI:
937 case AArch64::LDR_PXI:
938 return I->getFlag(MachineInstr::FrameSetup) ||
939 I->getFlag(MachineInstr::FrameDestroy);
943 void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
944 MachineBasicBlock &MBB) const {
945 MachineBasicBlock::iterator MBBI = MBB.begin();
946 const MachineFrameInfo &MFI = MF.getFrameInfo();
947 const Function &F = MF.getFunction();
948 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
949 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
950 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
951 MachineModuleInfo &MMI = MF.getMMI();
952 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
953 bool needsFrameMoves =
954 MF.needsFrameMoves() && !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
955 bool HasFP = hasFP(MF);
956 bool NeedsWinCFI = needsWinCFI(MF);
957 bool HasWinCFI = false;
958 auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
960 bool IsFunclet = MBB.isEHFuncletEntry();
962 // At this point, we're going to decide whether or not the function uses a
963 // redzone. In most cases, the function doesn't have a redzone so let's
964 // assume that's false and set it to true in the case that there's a redzone.
965 AFI->setHasRedZone(false);
967 // Debug location must be unknown since the first debug location is used
968 // to determine the end of the prologue.
971 if (ShouldSignReturnAddress(MF)) {
972 if (ShouldSignWithAKey(MF))
973 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
974 .setMIFlag(MachineInstr::FrameSetup);
976 BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
977 .setMIFlag(MachineInstr::FrameSetup);
978 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
979 .setMIFlag(MachineInstr::FrameSetup);
983 MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
984 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
985 .addCFIIndex(CFIIndex)
986 .setMIFlags(MachineInstr::FrameSetup);
989 // All calls are tail calls in GHC calling conv, and functions have no
990 // prologue/epilogue.
991 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
994 // Set tagged base pointer to the bottom of the stack frame.
995 // Ideally it should match SP value after prologue.
996 AFI->setTaggedBasePointerOffset(MFI.getStackSize());
998 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1000 // getStackSize() includes all the locals in its size calculation. We don't
1001 // include these locals when computing the stack size of a funclet, as they
1002 // are allocated in the parent's stack frame and accessed via the frame
1003 // pointer from the funclet. We only save the callee saved registers in the
1004 // funclet, which are really the callee saved registers of the parent
1005 // function, including the funclet.
1006 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1007 : MFI.getStackSize();
1008 if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
1009 assert(!HasFP && "unexpected function without stack frame but with FP");
1010 assert(!SVEStackSize &&
1011 "unexpected function without stack frame but with SVE objects");
1012 // All of the stack allocation is for locals.
1013 AFI->setLocalStackSize(NumBytes);
1016 // REDZONE: If the stack size is less than 128 bytes, we don't need
1017 // to actually allocate.
1018 if (canUseRedZone(MF)) {
1019 AFI->setHasRedZone(true);
1020 ++NumRedZoneFunctions;
1022 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1023 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
1024 false, NeedsWinCFI, &HasWinCFI);
1025 if (!NeedsWinCFI && needsFrameMoves) {
1026 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
1027 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
1028 // Encode the stack size of the leaf function.
1029 unsigned CFIIndex = MF.addFrameInst(
1030 MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
1031 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1032 .addCFIIndex(CFIIndex)
1033 .setMIFlags(MachineInstr::FrameSetup);
1039 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1040 .setMIFlag(MachineInstr::FrameSetup);
1047 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1048 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1050 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1051 // All of the remaining stack allocations are for locals.
1052 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1053 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1054 if (CombineSPBump) {
1055 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1056 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1057 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup, false,
1058 NeedsWinCFI, &HasWinCFI);
1060 } else if (PrologueSaveSize != 0) {
1061 MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
1062 MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
1063 NumBytes -= PrologueSaveSize;
1065 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1067 // Move past the saves of the callee-saved registers, fixing up the offsets
1068 // and pre-inc if we decided to combine the callee-save and local stack
1069 // pointer bump above.
1070 MachineBasicBlock::iterator End = MBB.end();
1071 while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1072 !IsSVECalleeSave(MBBI)) {
1074 fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
1075 NeedsWinCFI, &HasWinCFI);
1079 // For funclets the FP belongs to the containing function.
1080 if (!IsFunclet && HasFP) {
1081 // Only set up FP if we actually need to.
1082 int64_t FPOffset = isTargetDarwin(MF) ? (AFI->getCalleeSavedStackSize() - 16) : 0;
1085 FPOffset += AFI->getLocalStackSize();
1087 // Issue sub fp, sp, FPOffset or
1088 // mov fp,sp when FPOffset is zero.
1089 // Note: All stores of callee-saved registers are marked as "FrameSetup".
1090 // This code marks the instruction(s) that set the FP also.
1091 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1092 {FPOffset, MVT::i8}, TII, MachineInstr::FrameSetup, false,
1093 NeedsWinCFI, &HasWinCFI);
1096 if (windowsRequiresStackProbe(MF, NumBytes)) {
1097 uint64_t NumWords = NumBytes >> 4;
1100 // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1101 // exceed this amount. We need to move at most 2^24 - 1 into x15.
1102 // This is at most two instructions, MOVZ follwed by MOVK.
1103 // TODO: Fix to use multiple stack alloc unwind codes for stacks
1104 // exceeding 256MB in size.
1105 if (NumBytes >= (1 << 28))
1106 report_fatal_error("Stack size cannot exceed 256MB for stack "
1107 "unwinding purposes");
1109 uint32_t LowNumWords = NumWords & 0xFFFF;
1110 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
1111 .addImm(LowNumWords)
1112 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1113 .setMIFlag(MachineInstr::FrameSetup);
1114 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1115 .setMIFlag(MachineInstr::FrameSetup);
1116 if ((NumWords & 0xFFFF0000) != 0) {
1117 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
1118 .addReg(AArch64::X15)
1119 .addImm((NumWords & 0xFFFF0000) >> 16) // High half
1120 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16))
1121 .setMIFlag(MachineInstr::FrameSetup);
1122 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1123 .setMIFlag(MachineInstr::FrameSetup);
1126 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
1128 .setMIFlags(MachineInstr::FrameSetup);
1131 switch (MF.getTarget().getCodeModel()) {
1132 case CodeModel::Tiny:
1133 case CodeModel::Small:
1134 case CodeModel::Medium:
1135 case CodeModel::Kernel:
1136 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1137 .addExternalSymbol("__chkstk")
1138 .addReg(AArch64::X15, RegState::Implicit)
1139 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1140 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1141 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1142 .setMIFlags(MachineInstr::FrameSetup);
1145 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1146 .setMIFlag(MachineInstr::FrameSetup);
1149 case CodeModel::Large:
1150 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1151 .addReg(AArch64::X16, RegState::Define)
1152 .addExternalSymbol("__chkstk")
1153 .addExternalSymbol("__chkstk")
1154 .setMIFlags(MachineInstr::FrameSetup);
1157 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1158 .setMIFlag(MachineInstr::FrameSetup);
1161 BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
1162 .addReg(AArch64::X16, RegState::Kill)
1163 .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1164 .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
1165 .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
1166 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
1167 .setMIFlags(MachineInstr::FrameSetup);
1170 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1171 .setMIFlag(MachineInstr::FrameSetup);
1176 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1177 .addReg(AArch64::SP, RegState::Kill)
1178 .addReg(AArch64::X15, RegState::Kill)
1179 .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
1180 .setMIFlags(MachineInstr::FrameSetup);
1183 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1185 .setMIFlag(MachineInstr::FrameSetup);
1190 StackOffset AllocateBefore = SVEStackSize, AllocateAfter = {};
1191 MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
1193 // Process the SVE callee-saves to determine what space needs to be
1195 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1196 // Find callee save instructions in frame.
1197 CalleeSavesBegin = MBBI;
1198 assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
1199 while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
1201 CalleeSavesEnd = MBBI;
1203 AllocateBefore = {CalleeSavedSize, MVT::nxv1i8};
1204 AllocateAfter = SVEStackSize - AllocateBefore;
1207 // Allocate space for the callee saves (if any).
1208 emitFrameOffset(MBB, CalleeSavesBegin, DL, AArch64::SP, AArch64::SP,
1209 -AllocateBefore, TII,
1210 MachineInstr::FrameSetup);
1212 // Finally allocate remaining SVE stack space.
1213 emitFrameOffset(MBB, CalleeSavesEnd, DL, AArch64::SP, AArch64::SP,
1214 -AllocateAfter, TII,
1215 MachineInstr::FrameSetup);
1217 // Allocate space for the rest of the frame.
1219 // Alignment is required for the parent frame, not the funclet
1220 const bool NeedsRealignment =
1221 !IsFunclet && RegInfo->needsStackRealignment(MF);
1222 unsigned scratchSPReg = AArch64::SP;
1224 if (NeedsRealignment) {
1225 scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1226 assert(scratchSPReg != AArch64::NoRegister);
1229 // If we're a leaf function, try using the red zone.
1230 if (!canUseRedZone(MF))
1231 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1232 // the correct value here, as NumBytes also includes padding bytes,
1233 // which shouldn't be counted here.
1234 emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP,
1235 {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
1236 false, NeedsWinCFI, &HasWinCFI);
1238 if (NeedsRealignment) {
1239 const unsigned NrBitsToZero = Log2(MFI.getMaxAlign());
1240 assert(NrBitsToZero > 1);
1241 assert(scratchSPReg != AArch64::SP);
1243 // SUB X9, SP, NumBytes
1244 // -- X9 is temporary register, so shouldn't contain any live data here,
1245 // -- free to use. This is already produced by emitFrameOffset above.
1246 // AND SP, X9, 0b11111...0000
1247 // The logical immediates have a non-trivial encoding. The following
1248 // formula computes the encoded immediate with all ones but
1249 // NrBitsToZero zero bits as least significant bits.
1250 uint32_t andMaskEncoded = (1 << 12) // = N
1251 | ((64 - NrBitsToZero) << 6) // immr
1252 | ((64 - NrBitsToZero - 1) << 0); // imms
1254 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1255 .addReg(scratchSPReg, RegState::Kill)
1256 .addImm(andMaskEncoded);
1257 AFI->setStackRealigned(true);
1260 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1261 .addImm(NumBytes & andMaskEncoded)
1262 .setMIFlag(MachineInstr::FrameSetup);
1267 // If we need a base pointer, set it up here. It's whatever the value of the
1268 // stack pointer is at this point. Any variable size objects will be allocated
1269 // after this, so we can still use the base pointer to reference locals.
1271 // FIXME: Clarify FrameSetup flags here.
1272 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1274 // For funclets the BP belongs to the containing function.
1275 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
1276 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1280 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1281 .setMIFlag(MachineInstr::FrameSetup);
1285 // The very last FrameSetup instruction indicates the end of prologue. Emit a
1286 // SEH opcode indicating the prologue end.
1287 if (NeedsWinCFI && HasWinCFI) {
1288 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1289 .setMIFlag(MachineInstr::FrameSetup);
1292 // SEH funclets are passed the frame pointer in X1. If the parent
1293 // function uses the base register, then the base register is used
1294 // directly, and is not retrieved from X1.
1295 if (IsFunclet && F.hasPersonalityFn()) {
1296 EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
1297 if (isAsynchronousEHPersonality(Per)) {
1298 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
1299 .addReg(AArch64::X1)
1300 .setMIFlag(MachineInstr::FrameSetup);
1301 MBB.addLiveIn(AArch64::X1);
1305 if (needsFrameMoves) {
1306 const DataLayout &TD = MF.getDataLayout();
1307 const int StackGrowth = isTargetDarwin(MF)
1308 ? (2 * -TD.getPointerSize(0))
1309 : -AFI->getCalleeSavedStackSize();
1310 Register FramePtr = RegInfo->getFrameRegister(MF);
1311 // An example of the prologue:
1318 // .cfi_personality 155, ___gxx_personality_v0
1320 // .cfi_lsda 16, Lexception33
1322 // stp xa,bx, [sp, -#offset]!
1324 // stp x28, x27, [sp, #offset-32]
1325 // stp fp, lr, [sp, #offset-16]
1326 // add fp, sp, #offset - 16
1327 // sub sp, sp, #1360
1330 // +-------------------------------------------+
1331 // 10000 | ........ | ........ | ........ | ........ |
1332 // 10004 | ........ | ........ | ........ | ........ |
1333 // +-------------------------------------------+
1334 // 10008 | ........ | ........ | ........ | ........ |
1335 // 1000c | ........ | ........ | ........ | ........ |
1336 // +===========================================+
1337 // 10010 | X28 Register |
1338 // 10014 | X28 Register |
1339 // +-------------------------------------------+
1340 // 10018 | X27 Register |
1341 // 1001c | X27 Register |
1342 // +===========================================+
1343 // 10020 | Frame Pointer |
1344 // 10024 | Frame Pointer |
1345 // +-------------------------------------------+
1346 // 10028 | Link Register |
1347 // 1002c | Link Register |
1348 // +===========================================+
1349 // 10030 | ........ | ........ | ........ | ........ |
1350 // 10034 | ........ | ........ | ........ | ........ |
1351 // +-------------------------------------------+
1352 // 10038 | ........ | ........ | ........ | ........ |
1353 // 1003c | ........ | ........ | ........ | ........ |
1354 // +-------------------------------------------+
1356 // [sp] = 10030 :: >>initial value<<
1357 // sp = 10020 :: stp fp, lr, [sp, #-16]!
1358 // fp = sp == 10020 :: mov fp, sp
1359 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1360 // sp == 10010 :: >>final value<<
1362 // The frame pointer (w29) points to address 10020. If we use an offset of
1363 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1364 // for w27, and -32 for w28:
1367 // .cfi_def_cfa w29, 16
1369 // .cfi_offset w30, -8
1371 // .cfi_offset w29, -16
1373 // .cfi_offset w27, -24
1375 // .cfi_offset w28, -32
1378 // Define the current CFA rule to use the provided FP.
1379 unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1380 unsigned CFIIndex = MF.addFrameInst(
1381 MCCFIInstruction::cfiDefCfa(nullptr, Reg, FixedObject - StackGrowth));
1382 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1383 .addCFIIndex(CFIIndex)
1384 .setMIFlags(MachineInstr::FrameSetup);
1386 // Encode the stack size of the leaf function.
1387 unsigned CFIIndex = MF.addFrameInst(
1388 MCCFIInstruction::cfiDefCfaOffset(nullptr, MFI.getStackSize()));
1389 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1390 .addCFIIndex(CFIIndex)
1391 .setMIFlags(MachineInstr::FrameSetup);
1394 // Now emit the moves for whatever callee saved regs we have (including FP,
1395 // LR if those are saved).
1396 emitCalleeSavedFrameMoves(MBB, MBBI);
1400 static void InsertReturnAddressAuth(MachineFunction &MF,
1401 MachineBasicBlock &MBB) {
1402 if (!ShouldSignReturnAddress(MF))
1404 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1405 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1407 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1409 if (MBBI != MBB.end())
1410 DL = MBBI->getDebugLoc();
1412 // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1413 // this instruction can safely used for any v8a architecture.
1414 // From v8.3a onwards there are optimised authenticate LR and return
1415 // instructions, namely RETA{A,B}, that can be used instead.
1416 if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1417 MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1418 BuildMI(MBB, MBBI, DL,
1419 TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1420 .copyImplicitOps(*MBBI);
1425 TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1426 .setMIFlag(MachineInstr::FrameDestroy);
1430 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1431 switch (MI.getOpcode()) {
1434 case AArch64::CATCHRET:
1435 case AArch64::CLEANUPRET:
1440 void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
1441 MachineBasicBlock &MBB) const {
1442 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1443 MachineFrameInfo &MFI = MF.getFrameInfo();
1444 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1445 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1447 bool NeedsWinCFI = needsWinCFI(MF);
1448 bool HasWinCFI = false;
1449 bool IsFunclet = false;
1450 auto WinCFI = make_scope_exit([&]() {
1451 if (!MF.hasWinCFI())
1452 MF.setHasWinCFI(HasWinCFI);
1455 if (MBB.end() != MBBI) {
1456 DL = MBBI->getDebugLoc();
1457 IsFunclet = isFuncletReturnInstr(*MBBI);
1460 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1461 : MFI.getStackSize();
1462 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1464 // All calls are tail calls in GHC calling conv, and functions have no
1465 // prologue/epilogue.
1466 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
1469 // Initial and residual are named for consistency with the prologue. Note that
1470 // in the epilogue, the residual adjustment is executed first.
1471 uint64_t ArgumentPopSize = getArgumentPopSize(MF, MBB);
1473 // The stack frame should be like below,
1475 // ---------------------- ---
1477 // | BytesInStackArgArea| CalleeArgStackSize
1478 // | (NumReusableBytes) | (of tail call)
1481 // ---------------------| --- |
1483 // | CalleeSavedReg | | |
1484 // | (CalleeSavedStackSize)| | |
1486 // ---------------------| | NumBytes
1487 // | | StackSize (StackAdjustUp)
1488 // | LocalStackSize | | |
1489 // | (covering callee | | |
1492 // ---------------------- --- ---
1494 // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1495 // = StackSize + ArgumentPopSize
1497 // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1498 // it as the 2nd argument of AArch64ISD::TC_RETURN.
1500 auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1503 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1504 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1506 uint64_t AfterCSRPopSize = ArgumentPopSize;
1507 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1508 // We cannot rely on the local stack size set in emitPrologue if the function
1509 // has funclets, as funclets have different local stack size requirements, and
1510 // the current value set in emitPrologue may be that of the containing
1512 if (MF.hasEHFunclets())
1513 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1514 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
1515 // Assume we can't combine the last pop with the sp restore.
1517 if (!CombineSPBump && PrologueSaveSize != 0) {
1518 MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1519 while (AArch64InstrInfo::isSEHInstruction(*Pop))
1520 Pop = std::prev(Pop);
1521 // Converting the last ldp to a post-index ldp is valid only if the last
1522 // ldp's offset is 0.
1523 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1524 // If the offset is 0, convert it to a post-index ldp.
1525 if (OffsetOp.getImm() == 0)
1526 convertCalleeSaveRestoreToSPPrePostIncDec(
1527 MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
1529 // If not, make sure to emit an add after the last ldp.
1530 // We're doing this by transfering the size to be restored from the
1531 // adjustment *before* the CSR pops to the adjustment *after* the CSR
1533 AfterCSRPopSize += PrologueSaveSize;
1537 // Move past the restores of the callee-saved registers.
1538 // If we plan on combining the sp bump of the local stack size and the callee
1539 // save stack size, we might need to adjust the CSR save and restore offsets.
1540 MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
1541 MachineBasicBlock::iterator Begin = MBB.begin();
1542 while (LastPopI != Begin) {
1544 if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
1545 IsSVECalleeSave(LastPopI)) {
1548 } else if (CombineSPBump)
1549 fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
1550 NeedsWinCFI, &HasWinCFI);
1555 BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1556 .setMIFlag(MachineInstr::FrameDestroy);
1559 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1561 // If there is a single SP update, insert it before the ret and we're done.
1562 if (CombineSPBump) {
1563 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1564 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1565 {NumBytes + (int64_t)AfterCSRPopSize, MVT::i8}, TII,
1566 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1567 if (NeedsWinCFI && HasWinCFI)
1568 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1569 TII->get(AArch64::SEH_EpilogEnd))
1570 .setMIFlag(MachineInstr::FrameDestroy);
1574 NumBytes -= PrologueSaveSize;
1575 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1577 // Process the SVE callee-saves to determine what space needs to be
1579 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
1580 MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
1581 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1582 RestoreBegin = std::prev(RestoreEnd);;
1583 while (IsSVECalleeSave(RestoreBegin) &&
1584 RestoreBegin != MBB.begin())
1588 assert(IsSVECalleeSave(RestoreBegin) &&
1589 IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
1591 StackOffset CalleeSavedSizeAsOffset = {CalleeSavedSize, MVT::nxv1i8};
1592 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
1593 DeallocateAfter = CalleeSavedSizeAsOffset;
1596 // Deallocate the SVE area.
1598 if (AFI->isStackRealigned()) {
1599 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize())
1600 // Set SP to start of SVE callee-save area from which they can
1601 // be reloaded. The code below will deallocate the stack space
1602 // space by moving FP -> SP.
1603 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
1604 {-CalleeSavedSize, MVT::nxv1i8}, TII,
1605 MachineInstr::FrameDestroy);
1607 if (AFI->getSVECalleeSavedStackSize()) {
1608 // Deallocate the non-SVE locals first before we can deallocate (and
1609 // restore callee saves) from the SVE area.
1610 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1611 {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy);
1615 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1616 DeallocateBefore, TII, MachineInstr::FrameDestroy);
1618 emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
1619 DeallocateAfter, TII, MachineInstr::FrameDestroy);
1624 bool RedZone = canUseRedZone(MF);
1625 // If this was a redzone leaf function, we don't need to restore the
1626 // stack pointer (but we may need to pop stack args for fastcc).
1627 if (RedZone && AfterCSRPopSize == 0)
1630 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1631 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
1632 if (NoCalleeSaveRestore)
1633 StackRestoreBytes += AfterCSRPopSize;
1635 // If we were able to combine the local stack pop with the argument pop,
1637 bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1639 // If we're done after this, make sure to help the load store optimizer.
1641 adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1643 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1644 {StackRestoreBytes, MVT::i8}, TII,
1645 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1649 BuildMI(MBB, MBB.getFirstTerminator(), DL,
1650 TII->get(AArch64::SEH_EpilogEnd))
1651 .setMIFlag(MachineInstr::FrameDestroy);
1659 // Restore the original stack pointer.
1660 // FIXME: Rather than doing the math here, we should instead just use
1661 // non-post-indexed loads for the restores if we aren't actually going to
1662 // be able to save any instructions.
1663 if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
1664 int64_t OffsetToFrameRecord =
1665 isTargetDarwin(MF) ? (-(int64_t)AFI->getCalleeSavedStackSize() + 16) : 0;
1666 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1667 {OffsetToFrameRecord, MVT::i8},
1668 TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
1669 } else if (NumBytes)
1670 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1671 {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy, false,
1674 // This must be placed after the callee-save restore code because that code
1675 // assumes the SP is at the same location as it was after the callee-save save
1676 // code in the prologue.
1677 if (AfterCSRPopSize) {
1678 // Find an insertion point for the first ldp so that it goes before the
1679 // shadow call stack epilog instruction. This ensures that the restore of
1680 // lr from x18 is placed after the restore from sp.
1681 auto FirstSPPopI = MBB.getFirstTerminator();
1682 while (FirstSPPopI != Begin) {
1683 auto Prev = std::prev(FirstSPPopI);
1684 if (Prev->getOpcode() != AArch64::LDRXpre ||
1685 Prev->getOperand(0).getReg() == AArch64::SP)
1690 adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1692 emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1693 {(int64_t)AfterCSRPopSize, MVT::i8}, TII,
1694 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1696 if (NeedsWinCFI && HasWinCFI)
1697 BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1698 .setMIFlag(MachineInstr::FrameDestroy);
1700 MF.setHasWinCFI(HasWinCFI);
1703 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1704 /// debug info. It's the same as what we use for resolving the code-gen
1705 /// references for now. FIXME: This can go wrong when references are
1706 /// SP-relative and simple call frames aren't used.
1707 int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
1709 Register &FrameReg) const {
1710 return resolveFrameIndexReference(
1713 MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
1718 int AArch64FrameLowering::getNonLocalFrameIndexReference(
1719 const MachineFunction &MF, int FI) const {
1720 return getSEHFrameIndexOffset(MF, FI);
1723 static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset) {
1724 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1725 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1727 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1729 unsigned FixedObject =
1730 getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
1731 unsigned FPAdjust = isTargetDarwin(MF)
1732 ? 16 : AFI->getCalleeSavedStackSize(MF.getFrameInfo());
1733 return {ObjectOffset + FixedObject + FPAdjust, MVT::i8};
1736 static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset) {
1737 const auto &MFI = MF.getFrameInfo();
1738 return {ObjectOffset + (int64_t)MFI.getStackSize(), MVT::i8};
1741 int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
1743 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1744 MF.getSubtarget().getRegisterInfo());
1745 int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
1746 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1747 ? getFPOffset(MF, ObjectOffset).getBytes()
1748 : getStackOffset(MF, ObjectOffset).getBytes();
1751 StackOffset AArch64FrameLowering::resolveFrameIndexReference(
1752 const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
1753 bool ForSimm) const {
1754 const auto &MFI = MF.getFrameInfo();
1755 int64_t ObjectOffset = MFI.getObjectOffset(FI);
1756 bool isFixed = MFI.isFixedObjectIndex(FI);
1757 bool isSVE = MFI.getStackID(FI) == TargetStackID::SVEVector;
1758 return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
1762 StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
1763 const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
1764 Register &FrameReg, bool PreferFP, bool ForSimm) const {
1765 const auto &MFI = MF.getFrameInfo();
1766 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1767 MF.getSubtarget().getRegisterInfo());
1768 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1769 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1771 int64_t FPOffset = getFPOffset(MF, ObjectOffset).getBytes();
1772 int64_t Offset = getStackOffset(MF, ObjectOffset).getBytes();
1774 !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1776 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1778 // Use frame pointer to reference fixed objects. Use it for locals if
1779 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1780 // reliable as a base). Make sure useFPForScavengingIndex() does the
1781 // right thing for the emergency spill slot.
1783 if (AFI->hasStackFrame() && !isSVE) {
1784 // We shouldn't prefer using the FP when there is an SVE area
1785 // in between the FP and the non-SVE locals/spills.
1786 PreferFP &= !SVEStackSize;
1788 // Note: Keeping the following as multiple 'if' statements rather than
1789 // merging to a single expression for readability.
1791 // Argument access should always use the FP.
1794 } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1795 // References to the CSR area must use FP if we're re-aligning the stack
1796 // since the dynamically-sized alignment padding is between the SP/BP and
1798 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1800 } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1801 // If the FPOffset is negative and we're producing a signed immediate, we
1802 // have to keep in mind that the available offset range for negative
1803 // offsets is smaller than for positive ones. If an offset is available
1804 // via the FP and the SP, use whichever is closest.
1805 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
1806 PreferFP |= Offset > -FPOffset;
1808 if (MFI.hasVarSizedObjects()) {
1809 // If we have variable sized objects, we can use either FP or BP, as the
1810 // SP offset is unknown. We can use the base pointer if we have one and
1811 // FP is not preferred. If not, we're stuck with using FP.
1812 bool CanUseBP = RegInfo->hasBasePointer(MF);
1813 if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1815 else if (!CanUseBP) // Can't use BP. Forced to use FP.
1817 // else we can use BP and FP, but the offset from FP won't fit.
1818 // That will make us scavenge registers which we can probably avoid by
1819 // using BP. If it won't fit for BP either, we'll scavenge anyway.
1820 } else if (FPOffset >= 0) {
1821 // Use SP or FP, whichever gives us the best chance of the offset
1822 // being in range for direct access. If the FPOffset is positive,
1823 // that'll always be best, as the SP will be even further away.
1825 } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1826 // Funclets access the locals contained in the parent's stack frame
1827 // via the frame pointer, so we have to use the FP in the parent
1831 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1832 "Funclets should only be present on Win64");
1835 // We have the choice between FP and (SP or BP).
1836 if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1842 assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1843 "In the presence of dynamic stack pointer realignment, "
1844 "non-argument/CSR objects cannot be accessed through the frame pointer");
1847 int64_t OffsetToSVEArea =
1848 MFI.getStackSize() - AFI->getCalleeSavedStackSize();
1849 StackOffset FPOffset = {ObjectOffset, MVT::nxv1i8};
1850 StackOffset SPOffset = SVEStackSize +
1851 StackOffset(ObjectOffset, MVT::nxv1i8) +
1852 StackOffset(OffsetToSVEArea, MVT::i8);
1853 // Always use the FP for SVE spills if available and beneficial.
1855 (SPOffset.getBytes() ||
1856 FPOffset.getScalableBytes() < SPOffset.getScalableBytes() ||
1857 RegInfo->needsStackRealignment(MF))) {
1858 FrameReg = RegInfo->getFrameRegister(MF);
1862 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
1863 : (unsigned)AArch64::SP;
1867 StackOffset ScalableOffset = {};
1868 if (UseFP && !(isFixed || isCSR))
1869 ScalableOffset = -SVEStackSize;
1870 if (!UseFP && (isFixed || isCSR))
1871 ScalableOffset = SVEStackSize;
1874 FrameReg = RegInfo->getFrameRegister(MF);
1875 return StackOffset(FPOffset, MVT::i8) + ScalableOffset;
1878 // Use the base pointer if we have one.
1879 if (RegInfo->hasBasePointer(MF))
1880 FrameReg = RegInfo->getBaseRegister();
1882 assert(!MFI.hasVarSizedObjects() &&
1883 "Can't use SP when we have var sized objects.");
1884 FrameReg = AArch64::SP;
1885 // If we're using the red zone for this function, the SP won't actually
1886 // be adjusted, so the offsets will be negative. They're also all
1887 // within range of the signed 9-bit immediate instructions.
1888 if (canUseRedZone(MF))
1889 Offset -= AFI->getLocalStackSize();
1892 return StackOffset(Offset, MVT::i8) + ScalableOffset;
1895 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1896 // Do not set a kill flag on values that are also marked as live-in. This
1897 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1898 // callee saved registers.
1899 // Omitting the kill flags is conservatively correct even if the live-in
1900 // is not used after all.
1901 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1902 return getKillRegState(!IsLiveIn);
1905 static bool produceCompactUnwindFrame(MachineFunction &MF) {
1906 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1907 AttributeList Attrs = MF.getFunction().getAttributes();
1908 return Subtarget.isTargetMachO() &&
1909 !(Subtarget.getTargetLowering()->supportSwiftError() &&
1910 Attrs.hasAttrSomewhere(Attribute::SwiftError));
1913 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
1915 // If we are generating register pairs for a Windows function that requires
1916 // EH support, then pair consecutive registers only. There are no unwind
1917 // opcodes for saves/restores of non-consectuve register pairs.
1918 // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
1919 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1921 // TODO: LR can be paired with any register. We don't support this yet in
1922 // the MCLayer. We need to add support for the save_lrpair unwind code.
1923 if (Reg2 == AArch64::FP)
1927 if (Reg2 == Reg1 + 1)
1932 /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
1933 /// WindowsCFI requires that only consecutive registers can be paired.
1934 /// LR and FP need to be allocated together when the frame needs to save
1935 /// the frame-record. This means any other register pairing with LR is invalid.
1936 static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
1937 bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord) {
1939 return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI);
1941 // If we need to store the frame record, don't pair any register
1942 // with LR other than FP.
1943 if (NeedsFrameRecord)
1944 return Reg2 == AArch64::LR;
1951 struct RegPairInfo {
1952 unsigned Reg1 = AArch64::NoRegister;
1953 unsigned Reg2 = AArch64::NoRegister;
1956 enum RegType { GPR, FPR64, FPR128, PPR, ZPR } Type;
1958 RegPairInfo() = default;
1960 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1962 unsigned getScale() const {
1973 llvm_unreachable("Unsupported type");
1976 bool isScalable() const { return Type == PPR || Type == ZPR; }
1979 } // end anonymous namespace
1981 static void computeCalleeSaveRegisterPairs(
1982 MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
1983 const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
1984 bool &NeedShadowCallStackProlog, bool NeedsFrameRecord) {
1989 bool IsWindows = isTargetWindows(MF);
1990 bool NeedsWinCFI = needsWinCFI(MF);
1991 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1992 MachineFrameInfo &MFI = MF.getFrameInfo();
1993 CallingConv::ID CC = MF.getFunction().getCallingConv();
1994 unsigned Count = CSI.size();
1996 // MachO's compact unwind format relies on all registers being stored in
1998 assert((!produceCompactUnwindFrame(MF) ||
1999 CC == CallingConv::PreserveMost ||
2000 (Count & 1) == 0) &&
2001 "Odd number of callee-saved regs to spill!");
2002 int ByteOffset = AFI->getCalleeSavedStackSize();
2003 int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2004 // On Linux, we will have either one or zero non-paired register. On Windows
2005 // with CFI, we can have multiple unpaired registers in order to utilize the
2006 // available unwind codes. This flag assures that the alignment fixup is done
2007 // only once, as intened.
2008 bool FixupDone = false;
2009 for (unsigned i = 0; i < Count; ++i) {
2011 RPI.Reg1 = CSI[i].getReg();
2013 if (AArch64::GPR64RegClass.contains(RPI.Reg1))
2014 RPI.Type = RegPairInfo::GPR;
2015 else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
2016 RPI.Type = RegPairInfo::FPR64;
2017 else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
2018 RPI.Type = RegPairInfo::FPR128;
2019 else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
2020 RPI.Type = RegPairInfo::ZPR;
2021 else if (AArch64::PPRRegClass.contains(RPI.Reg1))
2022 RPI.Type = RegPairInfo::PPR;
2024 llvm_unreachable("Unsupported register class.");
2026 // Add the next reg to the pair if it is in the same register class.
2027 if (i + 1 < Count) {
2028 unsigned NextReg = CSI[i + 1].getReg();
2030 case RegPairInfo::GPR:
2031 if (AArch64::GPR64RegClass.contains(NextReg) &&
2032 !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows, NeedsWinCFI,
2036 case RegPairInfo::FPR64:
2037 if (AArch64::FPR64RegClass.contains(NextReg) &&
2038 !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
2041 case RegPairInfo::FPR128:
2042 if (AArch64::FPR128RegClass.contains(NextReg))
2045 case RegPairInfo::PPR:
2046 case RegPairInfo::ZPR:
2051 // If either of the registers to be saved is the lr register, it means that
2052 // we also need to save lr in the shadow call stack.
2053 if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
2054 MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
2055 if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
2056 report_fatal_error("Must reserve x18 to use shadow call stack");
2057 NeedShadowCallStackProlog = true;
2060 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
2061 // list to come in sorted by frame index so that we can issue the store
2062 // pair instructions directly. Assert if we see anything otherwise.
2064 // The order of the registers in the list is controlled by
2065 // getCalleeSavedRegs(), so they will always be in-order, as well.
2066 assert((!RPI.isPaired() ||
2067 (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
2068 "Out of order callee saved regs!");
2070 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2071 RPI.Reg1 == AArch64::LR) &&
2072 "FrameRecord must be allocated together with LR");
2074 // Windows AAPCS has FP and LR reversed.
2075 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2076 RPI.Reg2 == AArch64::LR) &&
2077 "FrameRecord must be allocated together with LR");
2079 // MachO's compact unwind format relies on all registers being stored in
2080 // adjacent register pairs.
2081 assert((!produceCompactUnwindFrame(MF) ||
2082 CC == CallingConv::PreserveMost ||
2084 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2085 RPI.Reg1 + 1 == RPI.Reg2))) &&
2086 "Callee-save registers not saved as adjacent register pair!");
2088 RPI.FrameIdx = CSI[i].getFrameIdx();
2090 int Scale = RPI.getScale();
2091 if (RPI.isScalable())
2092 ScalableByteOffset -= Scale;
2094 ByteOffset -= RPI.isPaired() ? 2 * Scale : Scale;
2096 assert(!(RPI.isScalable() && RPI.isPaired()) &&
2097 "Paired spill/fill instructions don't exist for SVE vectors");
2099 // Round up size of non-pair to pair size if we need to pad the
2100 // callee-save area to ensure 16-byte alignment.
2101 if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
2102 !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2106 assert(ByteOffset % 16 == 0);
2107 assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
2108 MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
2111 int Offset = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2112 assert(Offset % Scale == 0);
2113 RPI.Offset = Offset / Scale;
2115 assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2116 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2117 "Offset out of bounds for LDP/STP immediate");
2119 RegPairs.push_back(RPI);
2125 bool AArch64FrameLowering::spillCalleeSavedRegisters(
2126 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2127 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2128 MachineFunction &MF = *MBB.getParent();
2129 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2130 bool NeedsWinCFI = needsWinCFI(MF);
2132 SmallVector<RegPairInfo, 8> RegPairs;
2134 bool NeedShadowCallStackProlog = false;
2135 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2136 NeedShadowCallStackProlog, hasFP(MF));
2137 const MachineRegisterInfo &MRI = MF.getRegInfo();
2139 if (NeedShadowCallStackProlog) {
2140 // Shadow call stack prolog: str x30, [x18], #8
2141 BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
2142 .addReg(AArch64::X18, RegState::Define)
2143 .addReg(AArch64::LR)
2144 .addReg(AArch64::X18)
2146 .setMIFlag(MachineInstr::FrameSetup);
2149 BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
2150 .setMIFlag(MachineInstr::FrameSetup);
2152 if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
2153 // Emit a CFI instruction that causes 8 to be subtracted from the value of
2154 // x18 when unwinding past this frame.
2155 static const char CFIInst[] = {
2156 dwarf::DW_CFA_val_expression,
2159 static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
2160 static_cast<char>(-8) & 0x7f, // addend (sleb128)
2162 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
2163 nullptr, StringRef(CFIInst, sizeof(CFIInst))));
2164 BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
2165 .addCFIIndex(CFIIndex)
2166 .setMIFlag(MachineInstr::FrameSetup);
2169 // This instruction also makes x18 live-in to the entry block.
2170 MBB.addLiveIn(AArch64::X18);
2173 for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
2175 RegPairInfo RPI = *RPII;
2176 unsigned Reg1 = RPI.Reg1;
2177 unsigned Reg2 = RPI.Reg2;
2180 // Issue sequence of spills for cs regs. The first spill may be converted
2181 // to a pre-decrement store later by emitPrologue if the callee-save stack
2182 // area allocation can't be combined with the local stack area allocation.
2184 // stp x22, x21, [sp, #0] // addImm(+0)
2185 // stp x20, x19, [sp, #16] // addImm(+2)
2186 // stp fp, lr, [sp, #32] // addImm(+4)
2187 // Rationale: This sequence saves uop updates compared to a sequence of
2188 // pre-increment spills like stp xi,xj,[sp,#-16]!
2189 // Note: Similar rationale and sequence for restores in epilog.
2193 case RegPairInfo::GPR:
2194 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2196 Alignment = Align(8);
2198 case RegPairInfo::FPR64:
2199 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2201 Alignment = Align(8);
2203 case RegPairInfo::FPR128:
2204 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2206 Alignment = Align(16);
2208 case RegPairInfo::ZPR:
2209 StrOpc = AArch64::STR_ZXI;
2211 Alignment = Align(16);
2213 case RegPairInfo::PPR:
2214 StrOpc = AArch64::STR_PXI;
2216 Alignment = Align(2);
2219 LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2220 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2221 dbgs() << ") -> fi#(" << RPI.FrameIdx;
2222 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2225 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2226 "Windows unwdinding requires a consecutive (FP,LR) pair");
2227 // Windows unwind codes require consecutive registers if registers are
2228 // paired. Make the switch here, so that the code below will save (x,x+1)
2230 unsigned FrameIdxReg1 = RPI.FrameIdx;
2231 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2232 if (NeedsWinCFI && RPI.isPaired()) {
2233 std::swap(Reg1, Reg2);
2234 std::swap(FrameIdxReg1, FrameIdxReg2);
2236 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
2237 if (!MRI.isReserved(Reg1))
2238 MBB.addLiveIn(Reg1);
2239 if (RPI.isPaired()) {
2240 if (!MRI.isReserved(Reg2))
2241 MBB.addLiveIn(Reg2);
2242 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
2243 MIB.addMemOperand(MF.getMachineMemOperand(
2244 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2245 MachineMemOperand::MOStore, Size, Alignment));
2247 MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
2248 .addReg(AArch64::SP)
2249 .addImm(RPI.Offset) // [sp, #offset*scale],
2250 // where factor*scale is implicit
2251 .setMIFlag(MachineInstr::FrameSetup);
2252 MIB.addMemOperand(MF.getMachineMemOperand(
2253 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2254 MachineMemOperand::MOStore, Size, Alignment));
2256 InsertSEH(MIB, TII, MachineInstr::FrameSetup);
2258 // Update the StackIDs of the SVE stack slots.
2259 MachineFrameInfo &MFI = MF.getFrameInfo();
2260 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
2261 MFI.setStackID(RPI.FrameIdx, TargetStackID::SVEVector);
2267 bool AArch64FrameLowering::restoreCalleeSavedRegisters(
2268 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2269 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2270 MachineFunction &MF = *MBB.getParent();
2271 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2273 SmallVector<RegPairInfo, 8> RegPairs;
2274 bool NeedsWinCFI = needsWinCFI(MF);
2276 if (MI != MBB.end())
2277 DL = MI->getDebugLoc();
2279 bool NeedShadowCallStackProlog = false;
2280 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2281 NeedShadowCallStackProlog, hasFP(MF));
2283 auto EmitMI = [&](const RegPairInfo &RPI) {
2284 unsigned Reg1 = RPI.Reg1;
2285 unsigned Reg2 = RPI.Reg2;
2287 // Issue sequence of restores for cs regs. The last restore may be converted
2288 // to a post-increment load later by emitEpilogue if the callee-save stack
2289 // area allocation can't be combined with the local stack area allocation.
2291 // ldp fp, lr, [sp, #32] // addImm(+4)
2292 // ldp x20, x19, [sp, #16] // addImm(+2)
2293 // ldp x22, x21, [sp, #0] // addImm(+0)
2294 // Note: see comment in spillCalleeSavedRegisters()
2299 case RegPairInfo::GPR:
2300 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2302 Alignment = Align(8);
2304 case RegPairInfo::FPR64:
2305 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2307 Alignment = Align(8);
2309 case RegPairInfo::FPR128:
2310 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2312 Alignment = Align(16);
2314 case RegPairInfo::ZPR:
2315 LdrOpc = AArch64::LDR_ZXI;
2317 Alignment = Align(16);
2319 case RegPairInfo::PPR:
2320 LdrOpc = AArch64::LDR_PXI;
2322 Alignment = Align(2);
2325 LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2326 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2327 dbgs() << ") -> fi#(" << RPI.FrameIdx;
2328 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2331 // Windows unwind codes require consecutive registers if registers are
2332 // paired. Make the switch here, so that the code below will save (x,x+1)
2334 unsigned FrameIdxReg1 = RPI.FrameIdx;
2335 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2336 if (NeedsWinCFI && RPI.isPaired()) {
2337 std::swap(Reg1, Reg2);
2338 std::swap(FrameIdxReg1, FrameIdxReg2);
2340 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
2341 if (RPI.isPaired()) {
2342 MIB.addReg(Reg2, getDefRegState(true));
2343 MIB.addMemOperand(MF.getMachineMemOperand(
2344 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2345 MachineMemOperand::MOLoad, Size, Alignment));
2347 MIB.addReg(Reg1, getDefRegState(true))
2348 .addReg(AArch64::SP)
2349 .addImm(RPI.Offset) // [sp, #offset*scale]
2350 // where factor*scale is implicit
2351 .setMIFlag(MachineInstr::FrameDestroy);
2352 MIB.addMemOperand(MF.getMachineMemOperand(
2353 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2354 MachineMemOperand::MOLoad, Size, Alignment));
2356 InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
2359 // SVE objects are always restored in reverse order.
2360 for (const RegPairInfo &RPI : reverse(RegPairs))
2361 if (RPI.isScalable())
2364 if (ReverseCSRRestoreSeq) {
2365 for (const RegPairInfo &RPI : reverse(RegPairs))
2366 if (!RPI.isScalable())
2369 for (const RegPairInfo &RPI : RegPairs)
2370 if (!RPI.isScalable())
2373 if (NeedShadowCallStackProlog) {
2374 // Shadow call stack epilog: ldr x30, [x18, #-8]!
2375 BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
2376 .addReg(AArch64::X18, RegState::Define)
2377 .addReg(AArch64::LR, RegState::Define)
2378 .addReg(AArch64::X18)
2380 .setMIFlag(MachineInstr::FrameDestroy);
2386 void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
2387 BitVector &SavedRegs,
2388 RegScavenger *RS) const {
2389 // All calls are tail calls in GHC calling conv, and functions have no
2390 // prologue/epilogue.
2391 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
2394 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2395 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2396 MF.getSubtarget().getRegisterInfo());
2397 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2398 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2399 unsigned UnspilledCSGPR = AArch64::NoRegister;
2400 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2402 MachineFrameInfo &MFI = MF.getFrameInfo();
2403 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2405 unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2406 ? RegInfo->getBaseRegister()
2407 : (unsigned)AArch64::NoRegister;
2409 unsigned ExtraCSSpill = 0;
2410 // Figure out which callee-saved registers to save/restore.
2411 for (unsigned i = 0; CSRegs[i]; ++i) {
2412 const unsigned Reg = CSRegs[i];
2414 // Add the base pointer register to SavedRegs if it is callee-save.
2415 if (Reg == BasePointerReg)
2418 bool RegUsed = SavedRegs.test(Reg);
2419 unsigned PairedReg = AArch64::NoRegister;
2420 if (AArch64::GPR64RegClass.contains(Reg) ||
2421 AArch64::FPR64RegClass.contains(Reg) ||
2422 AArch64::FPR128RegClass.contains(Reg))
2423 PairedReg = CSRegs[i ^ 1];
2426 if (AArch64::GPR64RegClass.contains(Reg) &&
2427 !RegInfo->isReservedReg(MF, Reg)) {
2428 UnspilledCSGPR = Reg;
2429 UnspilledCSGPRPaired = PairedReg;
2434 // MachO's compact unwind format relies on all registers being stored in
2436 // FIXME: the usual format is actually better if unwinding isn't needed.
2437 if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2438 !SavedRegs.test(PairedReg)) {
2439 SavedRegs.set(PairedReg);
2440 if (AArch64::GPR64RegClass.contains(PairedReg) &&
2441 !RegInfo->isReservedReg(MF, PairedReg))
2442 ExtraCSSpill = PairedReg;
2446 if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
2447 !Subtarget.isTargetWindows()) {
2448 // For Windows calling convention on a non-windows OS, where X18 is treated
2449 // as reserved, back up X18 when entering non-windows code (marked with the
2450 // Windows calling convention) and restore when returning regardless of
2451 // whether the individual function uses it - it might call other functions
2453 SavedRegs.set(AArch64::X18);
2456 // Calculates the callee saved stack size.
2457 unsigned CSStackSize = 0;
2458 unsigned SVECSStackSize = 0;
2459 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2460 const MachineRegisterInfo &MRI = MF.getRegInfo();
2461 for (unsigned Reg : SavedRegs.set_bits()) {
2462 auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
2463 if (AArch64::PPRRegClass.contains(Reg) ||
2464 AArch64::ZPRRegClass.contains(Reg))
2465 SVECSStackSize += RegSize;
2467 CSStackSize += RegSize;
2470 // Save number of saved regs, so we can easily update CSStackSize later.
2471 unsigned NumSavedRegs = SavedRegs.count();
2473 // The frame record needs to be created by saving the appropriate registers
2474 uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
2476 windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2477 SavedRegs.set(AArch64::FP);
2478 SavedRegs.set(AArch64::LR);
2481 LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2483 : SavedRegs.set_bits()) dbgs()
2484 << ' ' << printReg(Reg, RegInfo);
2487 // If any callee-saved registers are used, the frame cannot be eliminated.
2488 int64_t SVEStackSize =
2489 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
2490 bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
2492 // The CSR spill slots have not been allocated yet, so estimateStackSize
2493 // won't include them.
2494 unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2496 // Conservatively always assume BigStack when there are SVE spills.
2497 bool BigStack = SVEStackSize ||
2498 (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2499 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2500 AFI->setHasStackFrame(true);
2502 // Estimate if we might need to scavenge a register at some point in order
2503 // to materialize a stack offset. If so, either spill one additional
2504 // callee-saved register or reserve a special spill slot to facilitate
2505 // register scavenging. If we already spilled an extra callee-saved register
2506 // above to keep the number of spills even, we don't need to do anything else
2509 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2510 LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2511 << " to get a scratch register.\n");
2512 SavedRegs.set(UnspilledCSGPR);
2513 // MachO's compact unwind format relies on all registers being stored in
2514 // pairs, so if we need to spill one extra for BigStack, then we need to
2516 if (produceCompactUnwindFrame(MF))
2517 SavedRegs.set(UnspilledCSGPRPaired);
2518 ExtraCSSpill = UnspilledCSGPR;
2521 // If we didn't find an extra callee-saved register to spill, create
2522 // an emergency spill slot.
2523 if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2524 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2525 const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2526 unsigned Size = TRI->getSpillSize(RC);
2527 Align Alignment = TRI->getSpillAlign(RC);
2528 int FI = MFI.CreateStackObject(Size, Alignment, false);
2529 RS->addScavengingFrameIndex(FI);
2530 LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2531 << " as the emergency spill slot.\n");
2535 // Adding the size of additional 64bit GPR saves.
2536 CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2537 uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
2538 LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2539 << EstimatedStackSize + AlignedCSStackSize
2542 assert((!MFI.isCalleeSavedInfoValid() ||
2543 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
2544 "Should not invalidate callee saved info");
2546 // Round up to register pair alignment to avoid additional SP adjustment
2548 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2549 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2550 AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
2553 bool AArch64FrameLowering::enableStackSlotScavenging(
2554 const MachineFunction &MF) const {
2555 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2556 return AFI->hasCalleeSaveStackFreeSpace();
2559 /// returns true if there are any SVE callee saves.
2560 static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
2561 int &Min, int &Max) {
2562 Min = std::numeric_limits<int>::max();
2563 Max = std::numeric_limits<int>::min();
2565 if (!MFI.isCalleeSavedInfoValid())
2568 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2569 for (auto &CS : CSI) {
2570 if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
2571 AArch64::PPRRegClass.contains(CS.getReg())) {
2572 assert((Max == std::numeric_limits<int>::min() ||
2573 Max + 1 == CS.getFrameIdx()) &&
2574 "SVE CalleeSaves are not consecutive");
2576 Min = std::min(Min, CS.getFrameIdx());
2577 Max = std::max(Max, CS.getFrameIdx());
2580 return Min != std::numeric_limits<int>::max();
2583 // Process all the SVE stack objects and determine offsets for each
2584 // object. If AssignOffsets is true, the offsets get assigned.
2585 // Fills in the first and last callee-saved frame indices into
2586 // Min/MaxCSFrameIndex, respectively.
2587 // Returns the size of the stack.
2588 static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
2589 int &MinCSFrameIndex,
2590 int &MaxCSFrameIndex,
2591 bool AssignOffsets) {
2593 // First process all fixed stack objects.
2594 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
2595 assert(MFI.getStackID(I) != TargetStackID::SVEVector &&
2596 "SVE vectors should never be passed on the stack by value, only by "
2600 auto Assign = [&MFI](int FI, int64_t Offset) {
2601 LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
2602 MFI.setObjectOffset(FI, Offset);
2607 // Then process all callee saved slots.
2608 if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
2609 // Assign offsets to the callee save slots.
2610 for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
2611 Offset += MFI.getObjectSize(I);
2612 Offset = alignTo(Offset, MFI.getObjectAlign(I));
2618 // Ensure that the Callee-save area is aligned to 16bytes.
2619 Offset = alignTo(Offset, Align(16U));
2621 // Create a buffer of SVE objects to allocate and sort it.
2622 SmallVector<int, 8> ObjectsToAllocate;
2623 for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
2624 unsigned StackID = MFI.getStackID(I);
2625 if (StackID != TargetStackID::SVEVector)
2627 if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
2629 if (MFI.isDeadObjectIndex(I))
2632 ObjectsToAllocate.push_back(I);
2635 // Allocate all SVE locals and spills
2636 for (unsigned FI : ObjectsToAllocate) {
2637 Align Alignment = MFI.getObjectAlign(FI);
2638 // FIXME: Given that the length of SVE vectors is not necessarily a power of
2639 // two, we'd need to align every object dynamically at runtime if the
2640 // alignment is larger than 16. This is not yet supported.
2641 if (Alignment > Align(16))
2643 "Alignment of scalable vectors > 16 bytes is not yet supported");
2645 Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
2647 Assign(FI, -Offset);
2653 int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
2654 MachineFrameInfo &MFI) const {
2655 int MinCSFrameIndex, MaxCSFrameIndex;
2656 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
2659 int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
2660 MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
2661 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
2665 void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
2666 MachineFunction &MF, RegScavenger *RS) const {
2667 MachineFrameInfo &MFI = MF.getFrameInfo();
2669 assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
2670 "Upwards growing stack unsupported");
2672 int MinCSFrameIndex, MaxCSFrameIndex;
2673 int64_t SVEStackSize =
2674 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
2676 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2677 AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
2678 AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
2680 // If this function isn't doing Win64-style C++ EH, we don't need to do
2682 if (!MF.hasEHFunclets())
2684 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2685 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2687 MachineBasicBlock &MBB = MF.front();
2688 auto MBBI = MBB.begin();
2689 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2692 // Create an UnwindHelp object.
2693 // The UnwindHelp object is allocated at the start of the fixed object area
2694 int64_t FixedObject =
2695 getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
2696 int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
2697 /*SPOffset*/ -FixedObject,
2698 /*IsImmutable=*/false);
2699 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2701 // We need to store -2 into the UnwindHelp object at the start of the
2704 RS->enterBasicBlockEnd(MBB);
2705 RS->backward(std::prev(MBBI));
2706 unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
2707 assert(DstReg && "There must be a free register after frame setup");
2708 BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2709 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2710 .addReg(DstReg, getKillRegState(true))
2711 .addFrameIndex(UnwindHelpFI)
2716 struct TagStoreInstr {
2718 int64_t Offset, Size;
2719 explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
2720 : MI(MI), Offset(Offset), Size(Size) {}
2723 class TagStoreEdit {
2724 MachineFunction *MF;
2725 MachineBasicBlock *MBB;
2726 MachineRegisterInfo *MRI;
2727 // Tag store instructions that are being replaced.
2728 SmallVector<TagStoreInstr, 8> TagStores;
2729 // Combined memref arguments of the above instructions.
2730 SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
2732 // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
2733 // FrameRegOffset + Size) with the address tag of SP.
2735 StackOffset FrameRegOffset;
2737 // If not None, move FrameReg to (FrameReg + FrameRegUpdate) at the end.
2738 Optional<int64_t> FrameRegUpdate;
2739 // MIFlags for any FrameReg updating instructions.
2740 unsigned FrameRegUpdateFlags;
2742 // Use zeroing instruction variants.
2746 void emitUnrolled(MachineBasicBlock::iterator InsertI);
2747 void emitLoop(MachineBasicBlock::iterator InsertI);
2750 TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
2751 : MBB(MBB), ZeroData(ZeroData) {
2752 MF = MBB->getParent();
2753 MRI = &MF->getRegInfo();
2755 // Add an instruction to be replaced. Instructions must be added in the
2756 // ascending order of Offset, and have to be adjacent.
2757 void addInstruction(TagStoreInstr I) {
2758 assert((TagStores.empty() ||
2759 TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
2760 "Non-adjacent tag store instructions.");
2761 TagStores.push_back(I);
2763 void clear() { TagStores.clear(); }
2764 // Emit equivalent code at the given location, and erase the current set of
2765 // instructions. May skip if the replacement is not profitable. May invalidate
2766 // the input iterator and replace it with a valid one.
2767 void emitCode(MachineBasicBlock::iterator &InsertI,
2768 const AArch64FrameLowering *TFI, bool IsLast);
2771 void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
2772 const AArch64InstrInfo *TII =
2773 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
2775 const int64_t kMinOffset = -256 * 16;
2776 const int64_t kMaxOffset = 255 * 16;
2778 Register BaseReg = FrameReg;
2779 int64_t BaseRegOffsetBytes = FrameRegOffset.getBytes();
2780 if (BaseRegOffsetBytes < kMinOffset ||
2781 BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset) {
2782 Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
2783 emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
2784 {BaseRegOffsetBytes, MVT::i8}, TII);
2785 BaseReg = ScratchReg;
2786 BaseRegOffsetBytes = 0;
2789 MachineInstr *LastI = nullptr;
2791 int64_t InstrSize = (Size > 16) ? 32 : 16;
2794 ? (ZeroData ? AArch64::STZGOffset : AArch64::STGOffset)
2795 : (ZeroData ? AArch64::STZ2GOffset : AArch64::ST2GOffset);
2796 MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
2797 .addReg(AArch64::SP)
2799 .addImm(BaseRegOffsetBytes / 16)
2800 .setMemRefs(CombinedMemRefs);
2801 // A store to [BaseReg, #0] should go last for an opportunity to fold the
2802 // final SP adjustment in the epilogue.
2803 if (BaseRegOffsetBytes == 0)
2805 BaseRegOffsetBytes += InstrSize;
2810 MBB->splice(InsertI, MBB, LastI);
2813 void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
2814 const AArch64InstrInfo *TII =
2815 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
2817 Register BaseReg = FrameRegUpdate
2819 : MRI->createVirtualRegister(&AArch64::GPR64RegClass);
2820 Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
2822 emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
2824 int64_t LoopSize = Size;
2825 // If the loop size is not a multiple of 32, split off one 16-byte store at
2826 // the end to fold BaseReg update into.
2827 if (FrameRegUpdate && *FrameRegUpdate)
2828 LoopSize -= LoopSize % 32;
2829 MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
2830 TII->get(ZeroData ? AArch64::STZGloop_wback
2831 : AArch64::STGloop_wback))
2836 .setMemRefs(CombinedMemRefs);
2838 LoopI->setFlags(FrameRegUpdateFlags);
2840 int64_t ExtraBaseRegUpdate =
2841 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getBytes() - Size) : 0;
2842 if (LoopSize < Size) {
2843 assert(FrameRegUpdate);
2844 assert(Size - LoopSize == 16);
2845 // Tag 16 more bytes at BaseReg and update BaseReg.
2846 BuildMI(*MBB, InsertI, DL,
2847 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
2851 .addImm(1 + ExtraBaseRegUpdate / 16)
2852 .setMemRefs(CombinedMemRefs)
2853 .setMIFlags(FrameRegUpdateFlags);
2854 } else if (ExtraBaseRegUpdate) {
2858 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
2861 .addImm(std::abs(ExtraBaseRegUpdate))
2863 .setMIFlags(FrameRegUpdateFlags);
2867 // Check if *II is a register update that can be merged into STGloop that ends
2868 // at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
2870 bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
2871 int64_t Size, int64_t *TotalOffset) {
2872 MachineInstr &MI = *II;
2873 if ((MI.getOpcode() == AArch64::ADDXri ||
2874 MI.getOpcode() == AArch64::SUBXri) &&
2875 MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
2876 unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
2877 int64_t Offset = MI.getOperand(2).getImm() << Shift;
2878 if (MI.getOpcode() == AArch64::SUBXri)
2880 int64_t AbsPostOffset = std::abs(Offset - Size);
2881 const int64_t kMaxOffset =
2882 0xFFF; // Max encoding for unshifted ADDXri / SUBXri
2883 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
2884 *TotalOffset = Offset;
2891 void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
2892 SmallVectorImpl<MachineMemOperand *> &MemRefs) {
2894 for (auto &TS : TSE) {
2895 MachineInstr *MI = TS.MI;
2896 // An instruction without memory operands may access anything. Be
2897 // conservative and return an empty list.
2898 if (MI->memoperands_empty()) {
2902 MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
2906 void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
2907 const AArch64FrameLowering *TFI, bool IsLast) {
2908 if (TagStores.empty())
2910 TagStoreInstr &FirstTagStore = TagStores[0];
2911 TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
2912 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
2913 DL = TagStores[0].MI->getDebugLoc();
2916 FrameRegOffset = TFI->resolveFrameOffsetReference(
2917 *MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
2918 /*PreferFP=*/false, /*ForSimm=*/true);
2920 FrameRegUpdate = None;
2922 mergeMemRefs(TagStores, CombinedMemRefs);
2924 LLVM_DEBUG(dbgs() << "Replacing adjacent STG instructions:\n";
2925 for (const auto &Instr
2926 : TagStores) { dbgs() << " " << *Instr.MI; });
2928 // Size threshold where a loop becomes shorter than a linear sequence of
2929 // tagging instructions.
2930 const int kSetTagLoopThreshold = 176;
2931 if (Size < kSetTagLoopThreshold) {
2932 if (TagStores.size() < 2)
2934 emitUnrolled(InsertI);
2936 MachineInstr *UpdateInstr = nullptr;
2937 int64_t TotalOffset;
2939 // See if we can merge base register update into the STGloop.
2940 // This is done in AArch64LoadStoreOptimizer for "normal" stores,
2941 // but STGloop is way too unusual for that, and also it only
2942 // realistically happens in function epilogue. Also, STGloop is expanded
2943 // before that pass.
2944 if (InsertI != MBB->end() &&
2945 canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getBytes() + Size,
2947 UpdateInstr = &*InsertI++;
2948 LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
2953 if (!UpdateInstr && TagStores.size() < 2)
2957 FrameRegUpdate = TotalOffset;
2958 FrameRegUpdateFlags = UpdateInstr->getFlags();
2962 UpdateInstr->eraseFromParent();
2965 for (auto &TS : TagStores)
2966 TS.MI->eraseFromParent();
2969 bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
2970 int64_t &Size, bool &ZeroData) {
2971 MachineFunction &MF = *MI.getParent()->getParent();
2972 const MachineFrameInfo &MFI = MF.getFrameInfo();
2974 unsigned Opcode = MI.getOpcode();
2975 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGOffset ||
2976 Opcode == AArch64::STZ2GOffset);
2978 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
2979 if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
2981 if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
2983 Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
2984 Size = MI.getOperand(2).getImm();
2988 if (Opcode == AArch64::STGOffset || Opcode == AArch64::STZGOffset)
2990 else if (Opcode == AArch64::ST2GOffset || Opcode == AArch64::STZ2GOffset)
2995 if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
2998 Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
2999 16 * MI.getOperand(2).getImm();
3003 // Detect a run of memory tagging instructions for adjacent stack frame slots,
3004 // and replace them with a shorter instruction sequence:
3005 // * replace STG + STG with ST2G
3006 // * replace STGloop + STGloop with STGloop
3007 // This code needs to run when stack slot offsets are already known, but before
3008 // FrameIndex operands in STG instructions are eliminated.
3009 MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
3010 const AArch64FrameLowering *TFI,
3013 int64_t Size, Offset;
3014 MachineInstr &MI = *II;
3015 MachineBasicBlock *MBB = MI.getParent();
3016 MachineBasicBlock::iterator NextI = ++II;
3017 if (&MI == &MBB->instr_back())
3019 if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
3022 SmallVector<TagStoreInstr, 4> Instrs;
3023 Instrs.emplace_back(&MI, Offset, Size);
3025 constexpr int kScanLimit = 10;
3027 for (MachineBasicBlock::iterator E = MBB->end();
3028 NextI != E && Count < kScanLimit; ++NextI) {
3029 MachineInstr &MI = *NextI;
3031 int64_t Size, Offset;
3032 // Collect instructions that update memory tags with a FrameIndex operand
3033 // and (when applicable) constant size, and whose output registers are dead
3034 // (the latter is almost always the case in practice). Since these
3035 // instructions effectively have no inputs or outputs, we are free to skip
3036 // any non-aliasing instructions in between without tracking used registers.
3037 if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
3038 if (ZeroData != FirstZeroData)
3040 Instrs.emplace_back(&MI, Offset, Size);
3044 // Only count non-transient, non-tagging instructions toward the scan
3046 if (!MI.isTransient())
3049 // Just in case, stop before the epilogue code starts.
3050 if (MI.getFlag(MachineInstr::FrameSetup) ||
3051 MI.getFlag(MachineInstr::FrameDestroy))
3054 // Reject anything that may alias the collected instructions.
3055 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
3059 // New code will be inserted after the last tagging instruction we've found.
3060 MachineBasicBlock::iterator InsertI = Instrs.back().MI;
3063 llvm::stable_sort(Instrs,
3064 [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
3065 return Left.Offset < Right.Offset;
3068 // Make sure that we don't have any overlapping stores.
3069 int64_t CurOffset = Instrs[0].Offset;
3070 for (auto &Instr : Instrs) {
3071 if (CurOffset > Instr.Offset)
3073 CurOffset = Instr.Offset + Instr.Size;
3076 // Find contiguous runs of tagged memory and emit shorter instruction
3077 // sequencies for them when possible.
3078 TagStoreEdit TSE(MBB, FirstZeroData);
3079 Optional<int64_t> EndOffset;
3080 for (auto &Instr : Instrs) {
3081 if (EndOffset && *EndOffset != Instr.Offset) {
3083 TSE.emitCode(InsertI, TFI, /*IsLast = */ false);
3087 TSE.addInstruction(Instr);
3088 EndOffset = Instr.Offset + Instr.Size;
3091 TSE.emitCode(InsertI, TFI, /*IsLast = */ true);
3097 void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
3098 MachineFunction &MF, RegScavenger *RS = nullptr) const {
3099 if (StackTaggingMergeSetTag)
3101 for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();)
3102 II = tryMergeAdjacentSTG(II, this, RS);
3105 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
3106 /// before the update. This is easily retrieved as it is exactly the offset
3107 /// that is set in processFunctionBeforeFrameFinalized.
3108 int AArch64FrameLowering::getFrameIndexReferencePreferSP(
3109 const MachineFunction &MF, int FI, Register &FrameReg,
3110 bool IgnoreSPUpdates) const {
3111 const MachineFrameInfo &MFI = MF.getFrameInfo();
3112 if (IgnoreSPUpdates) {
3113 LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
3114 << MFI.getObjectOffset(FI) << "\n");
3115 FrameReg = AArch64::SP;
3116 return MFI.getObjectOffset(FI);
3119 return getFrameIndexReference(MF, FI, FrameReg);
3122 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
3123 /// the parent's frame pointer
3124 unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
3125 const MachineFunction &MF) const {
3129 /// Funclets only need to account for space for the callee saved registers,
3130 /// as the locals are accounted for in the parent's stack frame.
3131 unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
3132 const MachineFunction &MF) const {
3133 // This is the size of the pushed CSRs.
3135 MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
3136 // This is the amount of stack a funclet needs to allocate.
3137 return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),