1 //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// This file implements the targeting of the InstructionSelector class for
12 /// \todo This should be generated by TableGen.
13 //===----------------------------------------------------------------------===//
15 #include "AArch64InstrInfo.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterBankInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "AArch64TargetMachine.h"
21 #include "MCTargetDesc/AArch64AddressingModes.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 #include "llvm/CodeGen/GlobalISel/Utils.h"
25 #include "llvm/CodeGen/MachineBasicBlock.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineInstr.h"
28 #include "llvm/CodeGen/MachineInstrBuilder.h"
29 #include "llvm/CodeGen/MachineOperand.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/Support/Debug.h"
33 #include "llvm/Support/raw_ostream.h"
35 #define DEBUG_TYPE "aarch64-isel"
41 #define GET_GLOBALISEL_PREDICATE_BITSET
42 #include "AArch64GenGlobalISel.inc"
43 #undef GET_GLOBALISEL_PREDICATE_BITSET
45 class AArch64InstructionSelector : public InstructionSelector {
47 AArch64InstructionSelector(const AArch64TargetMachine &TM,
48 const AArch64Subtarget &STI,
49 const AArch64RegisterBankInfo &RBI);
51 bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
52 static const char *getName() { return DEBUG_TYPE; }
55 /// tblgen-erated 'select' implementation, used as the initial selector for
56 /// the patterns that don't require complex C++.
57 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
59 bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
60 MachineRegisterInfo &MRI) const;
61 bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
62 MachineRegisterInfo &MRI) const;
64 bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
65 MachineRegisterInfo &MRI) const;
67 ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
69 ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
72 ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
73 return selectAddrModeUnscaled(Root, 1);
75 ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
76 return selectAddrModeUnscaled(Root, 2);
78 ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
79 return selectAddrModeUnscaled(Root, 4);
81 ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
82 return selectAddrModeUnscaled(Root, 8);
84 ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
85 return selectAddrModeUnscaled(Root, 16);
88 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
91 ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
92 return selectAddrModeIndexed(Root, Width / 8);
95 const AArch64TargetMachine &TM;
96 const AArch64Subtarget &STI;
97 const AArch64InstrInfo &TII;
98 const AArch64RegisterInfo &TRI;
99 const AArch64RegisterBankInfo &RBI;
101 #define GET_GLOBALISEL_PREDICATES_DECL
102 #include "AArch64GenGlobalISel.inc"
103 #undef GET_GLOBALISEL_PREDICATES_DECL
105 // We declare the temporaries used by selectImpl() in the class to minimize the
106 // cost of constructing placeholder values.
107 #define GET_GLOBALISEL_TEMPORARIES_DECL
108 #include "AArch64GenGlobalISel.inc"
109 #undef GET_GLOBALISEL_TEMPORARIES_DECL
112 } // end anonymous namespace
114 #define GET_GLOBALISEL_IMPL
115 #include "AArch64GenGlobalISel.inc"
116 #undef GET_GLOBALISEL_IMPL
118 AArch64InstructionSelector::AArch64InstructionSelector(
119 const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
120 const AArch64RegisterBankInfo &RBI)
121 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
122 TRI(*STI.getRegisterInfo()), RBI(RBI),
123 #define GET_GLOBALISEL_PREDICATES_INIT
124 #include "AArch64GenGlobalISel.inc"
125 #undef GET_GLOBALISEL_PREDICATES_INIT
126 #define GET_GLOBALISEL_TEMPORARIES_INIT
127 #include "AArch64GenGlobalISel.inc"
128 #undef GET_GLOBALISEL_TEMPORARIES_INIT
132 // FIXME: This should be target-independent, inferred from the types declared
133 // for each class in the bank.
134 static const TargetRegisterClass *
135 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
136 const RegisterBankInfo &RBI,
137 bool GetAllRegSet = false) {
138 if (RB.getID() == AArch64::GPRRegBankID) {
139 if (Ty.getSizeInBits() <= 32)
140 return GetAllRegSet ? &AArch64::GPR32allRegClass
141 : &AArch64::GPR32RegClass;
142 if (Ty.getSizeInBits() == 64)
143 return GetAllRegSet ? &AArch64::GPR64allRegClass
144 : &AArch64::GPR64RegClass;
148 if (RB.getID() == AArch64::FPRRegBankID) {
149 if (Ty.getSizeInBits() <= 16)
150 return &AArch64::FPR16RegClass;
151 if (Ty.getSizeInBits() == 32)
152 return &AArch64::FPR32RegClass;
153 if (Ty.getSizeInBits() == 64)
154 return &AArch64::FPR64RegClass;
155 if (Ty.getSizeInBits() == 128)
156 return &AArch64::FPR128RegClass;
163 /// Check whether \p I is a currently unsupported binary operation:
164 /// - it has an unsized type
165 /// - an operand is not a vreg
166 /// - all operands are not in the same bank
167 /// These are checks that should someday live in the verifier, but right now,
168 /// these are mostly limitations of the aarch64 selector.
169 static bool unsupportedBinOp(const MachineInstr &I,
170 const AArch64RegisterBankInfo &RBI,
171 const MachineRegisterInfo &MRI,
172 const AArch64RegisterInfo &TRI) {
173 LLT Ty = MRI.getType(I.getOperand(0).getReg());
175 DEBUG(dbgs() << "Generic binop register should be typed\n");
179 const RegisterBank *PrevOpBank = nullptr;
180 for (auto &MO : I.operands()) {
181 // FIXME: Support non-register operands.
183 DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
187 // FIXME: Can generic operations have physical registers operands? If
188 // so, this will need to be taught about that, and we'll need to get the
189 // bank out of the minimal class for the register.
190 // Either way, this needs to be documented (and possibly verified).
191 if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
192 DEBUG(dbgs() << "Generic inst has physical register operand\n");
196 const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
198 DEBUG(dbgs() << "Generic register has no bank or class\n");
202 if (PrevOpBank && OpBank != PrevOpBank) {
203 DEBUG(dbgs() << "Generic inst operands have different banks\n");
211 /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
212 /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
213 /// and of size \p OpSize.
214 /// \returns \p GenericOpc if the combination is unsupported.
215 static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
218 case AArch64::GPRRegBankID:
220 switch (GenericOpc) {
221 case TargetOpcode::G_SHL:
222 return AArch64::LSLVWr;
223 case TargetOpcode::G_LSHR:
224 return AArch64::LSRVWr;
225 case TargetOpcode::G_ASHR:
226 return AArch64::ASRVWr;
230 } else if (OpSize == 64) {
231 switch (GenericOpc) {
232 case TargetOpcode::G_GEP:
233 return AArch64::ADDXrr;
234 case TargetOpcode::G_SHL:
235 return AArch64::LSLVXr;
236 case TargetOpcode::G_LSHR:
237 return AArch64::LSRVXr;
238 case TargetOpcode::G_ASHR:
239 return AArch64::ASRVXr;
245 case AArch64::FPRRegBankID:
248 switch (GenericOpc) {
249 case TargetOpcode::G_FADD:
250 return AArch64::FADDSrr;
251 case TargetOpcode::G_FSUB:
252 return AArch64::FSUBSrr;
253 case TargetOpcode::G_FMUL:
254 return AArch64::FMULSrr;
255 case TargetOpcode::G_FDIV:
256 return AArch64::FDIVSrr;
261 switch (GenericOpc) {
262 case TargetOpcode::G_FADD:
263 return AArch64::FADDDrr;
264 case TargetOpcode::G_FSUB:
265 return AArch64::FSUBDrr;
266 case TargetOpcode::G_FMUL:
267 return AArch64::FMULDrr;
268 case TargetOpcode::G_FDIV:
269 return AArch64::FDIVDrr;
270 case TargetOpcode::G_OR:
271 return AArch64::ORRv8i8;
281 /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
282 /// appropriate for the (value) register bank \p RegBankID and of memory access
283 /// size \p OpSize. This returns the variant with the base+unsigned-immediate
284 /// addressing mode (e.g., LDRXui).
285 /// \returns \p GenericOpc if the combination is unsupported.
286 static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
288 const bool isStore = GenericOpc == TargetOpcode::G_STORE;
290 case AArch64::GPRRegBankID:
293 return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
295 return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
297 return isStore ? AArch64::STRWui : AArch64::LDRWui;
299 return isStore ? AArch64::STRXui : AArch64::LDRXui;
302 case AArch64::FPRRegBankID:
305 return isStore ? AArch64::STRBui : AArch64::LDRBui;
307 return isStore ? AArch64::STRHui : AArch64::LDRHui;
309 return isStore ? AArch64::STRSui : AArch64::LDRSui;
311 return isStore ? AArch64::STRDui : AArch64::LDRDui;
318 static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
319 MachineRegisterInfo &MRI, unsigned SrcReg) {
320 // Copies from gpr32 to fpr16 need to use a sub-register copy.
321 unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
322 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
325 unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
326 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
328 .addUse(CopyReg, 0, AArch64::hsub);
330 MachineOperand &RegOp = I.getOperand(1);
331 RegOp.setReg(SubRegCopy);
335 static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
336 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
337 const RegisterBankInfo &RBI) {
339 unsigned DstReg = I.getOperand(0).getReg();
340 unsigned SrcReg = I.getOperand(1).getReg();
342 if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
343 if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
344 !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
345 const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
346 const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
347 MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
348 if (SrcRC == &AArch64::GPR32allRegClass)
349 return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
351 assert(I.isCopy() && "Generic operators do not allow physical registers");
355 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
356 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
358 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
360 assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
361 "No phys reg on generic operators");
363 (DstSize == SrcSize ||
364 // Copies are a mean to setup initial types, the number of
365 // bits may not exactly match.
366 (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
367 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
368 // Copies are a mean to copy bits around, as long as we are
369 // on the same register class, that's fine. Otherwise, that
370 // means we need some SUBREG_TO_REG or AND & co.
371 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
372 "Copy with different width?!");
373 assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
374 "GPRs cannot get more than 64-bit width values");
376 const TargetRegisterClass *RC = getRegClassForTypeOnBank(
377 MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
379 DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
383 if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
384 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
385 const TargetRegisterClass *SrcRC =
386 RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
387 const RegisterBank *RB = nullptr;
389 RB = RegClassOrBank.get<const RegisterBank *>();
390 SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
392 // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
393 if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
394 unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
395 BuildMI(*I.getParent(), I, I.getDebugLoc(),
396 TII.get(AArch64::SUBREG_TO_REG))
400 .addImm(AArch64::hsub);
401 MachineOperand &RegOp = I.getOperand(1);
402 RegOp.setReg(PromoteReg);
403 } else if (RC == &AArch64::FPR16RegClass &&
404 SrcRC == &AArch64::GPR32allRegClass) {
405 selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
409 // No need to constrain SrcReg. It will get constrained when
410 // we hit another of its use or its defs.
411 // Copies do not have constraints.
412 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
413 DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
417 I.setDesc(TII.get(AArch64::COPY));
421 static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
422 if (!DstTy.isScalar() || !SrcTy.isScalar())
425 const unsigned DstSize = DstTy.getSizeInBits();
426 const unsigned SrcSize = SrcTy.getSizeInBits();
432 switch (GenericOpc) {
433 case TargetOpcode::G_SITOFP:
434 return AArch64::SCVTFUWSri;
435 case TargetOpcode::G_UITOFP:
436 return AArch64::UCVTFUWSri;
437 case TargetOpcode::G_FPTOSI:
438 return AArch64::FCVTZSUWSr;
439 case TargetOpcode::G_FPTOUI:
440 return AArch64::FCVTZUUWSr;
445 switch (GenericOpc) {
446 case TargetOpcode::G_SITOFP:
447 return AArch64::SCVTFUXSri;
448 case TargetOpcode::G_UITOFP:
449 return AArch64::UCVTFUXSri;
450 case TargetOpcode::G_FPTOSI:
451 return AArch64::FCVTZSUWDr;
452 case TargetOpcode::G_FPTOUI:
453 return AArch64::FCVTZUUWDr;
463 switch (GenericOpc) {
464 case TargetOpcode::G_SITOFP:
465 return AArch64::SCVTFUWDri;
466 case TargetOpcode::G_UITOFP:
467 return AArch64::UCVTFUWDri;
468 case TargetOpcode::G_FPTOSI:
469 return AArch64::FCVTZSUXSr;
470 case TargetOpcode::G_FPTOUI:
471 return AArch64::FCVTZUUXSr;
476 switch (GenericOpc) {
477 case TargetOpcode::G_SITOFP:
478 return AArch64::SCVTFUXDri;
479 case TargetOpcode::G_UITOFP:
480 return AArch64::UCVTFUXDri;
481 case TargetOpcode::G_FPTOSI:
482 return AArch64::FCVTZSUXDr;
483 case TargetOpcode::G_FPTOUI:
484 return AArch64::FCVTZUUXDr;
497 static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
500 llvm_unreachable("Unknown condition code!");
501 case CmpInst::ICMP_NE:
502 return AArch64CC::NE;
503 case CmpInst::ICMP_EQ:
504 return AArch64CC::EQ;
505 case CmpInst::ICMP_SGT:
506 return AArch64CC::GT;
507 case CmpInst::ICMP_SGE:
508 return AArch64CC::GE;
509 case CmpInst::ICMP_SLT:
510 return AArch64CC::LT;
511 case CmpInst::ICMP_SLE:
512 return AArch64CC::LE;
513 case CmpInst::ICMP_UGT:
514 return AArch64CC::HI;
515 case CmpInst::ICMP_UGE:
516 return AArch64CC::HS;
517 case CmpInst::ICMP_ULT:
518 return AArch64CC::LO;
519 case CmpInst::ICMP_ULE:
520 return AArch64CC::LS;
524 static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
525 AArch64CC::CondCode &CondCode,
526 AArch64CC::CondCode &CondCode2) {
527 CondCode2 = AArch64CC::AL;
530 llvm_unreachable("Unknown FP condition!");
531 case CmpInst::FCMP_OEQ:
532 CondCode = AArch64CC::EQ;
534 case CmpInst::FCMP_OGT:
535 CondCode = AArch64CC::GT;
537 case CmpInst::FCMP_OGE:
538 CondCode = AArch64CC::GE;
540 case CmpInst::FCMP_OLT:
541 CondCode = AArch64CC::MI;
543 case CmpInst::FCMP_OLE:
544 CondCode = AArch64CC::LS;
546 case CmpInst::FCMP_ONE:
547 CondCode = AArch64CC::MI;
548 CondCode2 = AArch64CC::GT;
550 case CmpInst::FCMP_ORD:
551 CondCode = AArch64CC::VC;
553 case CmpInst::FCMP_UNO:
554 CondCode = AArch64CC::VS;
556 case CmpInst::FCMP_UEQ:
557 CondCode = AArch64CC::EQ;
558 CondCode2 = AArch64CC::VS;
560 case CmpInst::FCMP_UGT:
561 CondCode = AArch64CC::HI;
563 case CmpInst::FCMP_UGE:
564 CondCode = AArch64CC::PL;
566 case CmpInst::FCMP_ULT:
567 CondCode = AArch64CC::LT;
569 case CmpInst::FCMP_ULE:
570 CondCode = AArch64CC::LE;
572 case CmpInst::FCMP_UNE:
573 CondCode = AArch64CC::NE;
578 bool AArch64InstructionSelector::selectCompareBranch(
579 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
581 const unsigned CondReg = I.getOperand(0).getReg();
582 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
583 MachineInstr *CCMI = MRI.getVRegDef(CondReg);
584 if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
585 CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
586 if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
589 unsigned LHS = CCMI->getOperand(2).getReg();
590 unsigned RHS = CCMI->getOperand(3).getReg();
591 if (!getConstantVRegVal(RHS, MRI))
594 const auto RHSImm = getConstantVRegVal(RHS, MRI);
595 if (!RHSImm || *RHSImm != 0)
598 const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
599 if (RB.getID() != AArch64::GPRRegBankID)
602 const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
603 if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
606 const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
609 CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
610 else if (CmpWidth == 64)
611 CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
615 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
619 constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
624 bool AArch64InstructionSelector::selectVaStartAAPCS(
625 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
629 bool AArch64InstructionSelector::selectVaStartDarwin(
630 MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
631 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
632 unsigned ListReg = I.getOperand(0).getReg();
634 unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
637 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
639 .addFrameIndex(FuncInfo->getVarArgsStackIndex())
643 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
645 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
649 .addMemOperand(*I.memoperands_begin());
651 constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
656 bool AArch64InstructionSelector::select(MachineInstr &I,
657 CodeGenCoverage &CoverageInfo) const {
658 assert(I.getParent() && "Instruction should be in a basic block!");
659 assert(I.getParent()->getParent() && "Instruction should be in a function!");
661 MachineBasicBlock &MBB = *I.getParent();
662 MachineFunction &MF = *MBB.getParent();
663 MachineRegisterInfo &MRI = MF.getRegInfo();
665 unsigned Opcode = I.getOpcode();
666 // G_PHI requires same handling as PHI
667 if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
668 // Certain non-generic instructions also need some special handling.
670 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
671 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
673 if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
674 const unsigned DefReg = I.getOperand(0).getReg();
675 const LLT DefTy = MRI.getType(DefReg);
677 const TargetRegisterClass *DefRC = nullptr;
678 if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
679 DefRC = TRI.getRegClass(DefReg);
681 const RegClassOrRegBank &RegClassOrBank =
682 MRI.getRegClassOrRegBank(DefReg);
684 DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
686 if (!DefTy.isValid()) {
687 DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
690 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
691 DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
693 DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
698 I.setDesc(TII.get(TargetOpcode::PHI));
700 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
704 return selectCopy(I, TII, MRI, TRI, RBI);
710 if (I.getNumOperands() != I.getNumExplicitOperands()) {
711 DEBUG(dbgs() << "Generic instruction has unexpected implicit operands\n");
715 if (selectImpl(I, CoverageInfo))
719 I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
722 case TargetOpcode::G_BRCOND: {
723 if (Ty.getSizeInBits() > 32) {
724 // We shouldn't need this on AArch64, but it would be implemented as an
725 // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
726 // bit being tested is < 32.
727 DEBUG(dbgs() << "G_BRCOND has type: " << Ty
728 << ", expected at most 32-bits");
732 const unsigned CondReg = I.getOperand(0).getReg();
733 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
735 if (selectCompareBranch(I, MF, MRI))
738 auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
740 .addImm(/*bit offset=*/0)
744 return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
747 case TargetOpcode::G_BRINDIRECT: {
748 I.setDesc(TII.get(AArch64::BR));
749 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
752 case TargetOpcode::G_FCONSTANT:
753 case TargetOpcode::G_CONSTANT: {
754 const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
756 const LLT s32 = LLT::scalar(32);
757 const LLT s64 = LLT::scalar(64);
758 const LLT p0 = LLT::pointer(0, 64);
760 const unsigned DefReg = I.getOperand(0).getReg();
761 const LLT DefTy = MRI.getType(DefReg);
762 const unsigned DefSize = DefTy.getSizeInBits();
763 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
765 // FIXME: Redundant check, but even less readable when factored out.
767 if (Ty != s32 && Ty != s64) {
768 DEBUG(dbgs() << "Unable to materialize FP " << Ty
769 << " constant, expected: " << s32 << " or " << s64
774 if (RB.getID() != AArch64::FPRRegBankID) {
775 DEBUG(dbgs() << "Unable to materialize FP " << Ty
776 << " constant on bank: " << RB << ", expected: FPR\n");
780 // The case when we have 0.0 is covered by tablegen. Reject it here so we
781 // can be sure tablegen works correctly and isn't rescued by this code.
782 if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
785 // s32 and s64 are covered by tablegen.
787 DEBUG(dbgs() << "Unable to materialize integer " << Ty
788 << " constant, expected: " << s32 << ", " << s64 << ", or "
793 if (RB.getID() != AArch64::GPRRegBankID) {
794 DEBUG(dbgs() << "Unable to materialize integer " << Ty
795 << " constant on bank: " << RB << ", expected: GPR\n");
800 const unsigned MovOpc =
801 DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
803 I.setDesc(TII.get(MovOpc));
806 const TargetRegisterClass &GPRRC =
807 DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
808 const TargetRegisterClass &FPRRC =
809 DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
811 const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
812 MachineOperand &RegOp = I.getOperand(0);
813 RegOp.setReg(DefGPRReg);
815 BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
816 TII.get(AArch64::COPY))
820 if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
821 DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
825 MachineOperand &ImmOp = I.getOperand(1);
826 // FIXME: Is going through int64_t always correct?
827 ImmOp.ChangeToImmediate(
828 ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
829 } else if (I.getOperand(1).isCImm()) {
830 uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
831 I.getOperand(1).ChangeToImmediate(Val);
832 } else if (I.getOperand(1).isImm()) {
833 uint64_t Val = I.getOperand(1).getImm();
834 I.getOperand(1).ChangeToImmediate(Val);
837 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
840 case TargetOpcode::G_EXTRACT: {
841 LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
842 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
844 unsigned SrcSize = SrcTy.getSizeInBits();
845 // Larger extracts are vectors, same-size extracts should be something else
846 // by now (either split up or simplified to a COPY).
847 if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
850 I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
851 MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
852 Ty.getSizeInBits() - 1);
855 assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
856 "unexpected G_EXTRACT types");
857 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
860 unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
861 BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
862 TII.get(AArch64::COPY))
863 .addDef(I.getOperand(0).getReg())
864 .addUse(DstReg, 0, AArch64::sub_32);
865 RBI.constrainGenericRegister(I.getOperand(0).getReg(),
866 AArch64::GPR32RegClass, MRI);
867 I.getOperand(0).setReg(DstReg);
869 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
872 case TargetOpcode::G_INSERT: {
873 LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
874 LLT DstTy = MRI.getType(I.getOperand(0).getReg());
875 unsigned DstSize = DstTy.getSizeInBits();
877 // Larger inserts are vectors, same-size ones should be something else by
878 // now (split up or turned into COPYs).
879 if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
882 I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
883 unsigned LSB = I.getOperand(3).getImm();
884 unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
885 I.getOperand(3).setImm((DstSize - LSB) % DstSize);
886 MachineInstrBuilder(MF, I).addImm(Width - 1);
889 assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
890 "unexpected G_INSERT types");
891 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
894 unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
895 BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
896 TII.get(AArch64::SUBREG_TO_REG))
899 .addUse(I.getOperand(2).getReg())
900 .addImm(AArch64::sub_32);
901 RBI.constrainGenericRegister(I.getOperand(2).getReg(),
902 AArch64::GPR32RegClass, MRI);
903 I.getOperand(2).setReg(SrcReg);
905 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
907 case TargetOpcode::G_FRAME_INDEX: {
908 // allocas and G_FRAME_INDEX are only supported in addrspace(0).
909 if (Ty != LLT::pointer(0, 64)) {
910 DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
911 << ", expected: " << LLT::pointer(0, 64) << '\n');
914 I.setDesc(TII.get(AArch64::ADDXri));
916 // MOs for a #0 shifted immediate.
917 I.addOperand(MachineOperand::CreateImm(0));
918 I.addOperand(MachineOperand::CreateImm(0));
920 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
923 case TargetOpcode::G_GLOBAL_VALUE: {
924 auto GV = I.getOperand(1).getGlobal();
925 if (GV->isThreadLocal()) {
926 // FIXME: we don't support TLS yet.
929 unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
930 if (OpFlags & AArch64II::MO_GOT) {
931 I.setDesc(TII.get(AArch64::LOADgot));
932 I.getOperand(1).setTargetFlags(OpFlags);
933 } else if (TM.getCodeModel() == CodeModel::Large) {
934 // Materialize the global using movz/movk instructions.
935 unsigned MovZDstReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
936 auto InsertPt = std::next(I.getIterator());
938 BuildMI(MBB, InsertPt, I.getDebugLoc(), TII.get(AArch64::MOVZXi))
940 MovZ->addOperand(MF, I.getOperand(1));
941 MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
943 MovZ->addOperand(MF, MachineOperand::CreateImm(0));
944 constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
946 auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags,
947 unsigned Offset, unsigned ForceDstReg) {
949 ForceDstReg ? ForceDstReg
950 : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
951 auto MovI = BuildMI(MBB, InsertPt, MovZ->getDebugLoc(),
952 TII.get(AArch64::MOVKXi))
955 MovI->addOperand(MF, MachineOperand::CreateGA(
956 GV, MovZ->getOperand(1).getOffset(), Flags));
957 MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
958 constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
961 unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
962 AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
963 DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
964 BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
968 I.setDesc(TII.get(AArch64::MOVaddr));
969 I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
970 MachineInstrBuilder MIB(MF, I);
971 MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
972 OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
974 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
977 case TargetOpcode::G_LOAD:
978 case TargetOpcode::G_STORE: {
980 LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
982 if (PtrTy != LLT::pointer(0, 64)) {
983 DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
984 << ", expected: " << LLT::pointer(0, 64) << '\n');
988 auto &MemOp = **I.memoperands_begin();
989 if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
990 DEBUG(dbgs() << "Atomic load/store not supported yet\n");
994 // FIXME: PR36018: Volatile loads in some cases are incorrectly selected by
995 // folding with an extend. Until we have a G_SEXTLOAD solution bail out if
997 if (Opcode == TargetOpcode::G_LOAD && MemOp.isVolatile())
1000 const unsigned PtrReg = I.getOperand(1).getReg();
1002 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
1003 // Sanity-check the pointer register.
1004 assert(PtrRB.getID() == AArch64::GPRRegBankID &&
1005 "Load/Store pointer operand isn't a GPR");
1006 assert(MRI.getType(PtrReg).isPointer() &&
1007 "Load/Store pointer operand isn't a pointer");
1010 const unsigned ValReg = I.getOperand(0).getReg();
1011 const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
1013 const unsigned NewOpc =
1014 selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemTy.getSizeInBits());
1015 if (NewOpc == I.getOpcode())
1018 I.setDesc(TII.get(NewOpc));
1020 uint64_t Offset = 0;
1021 auto *PtrMI = MRI.getVRegDef(PtrReg);
1023 // Try to fold a GEP into our unsigned immediate addressing mode.
1024 if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
1025 if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
1026 int64_t Imm = *COff;
1027 const unsigned Size = MemTy.getSizeInBits() / 8;
1028 const unsigned Scale = Log2_32(Size);
1029 if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
1030 unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
1031 I.getOperand(1).setReg(Ptr2Reg);
1032 PtrMI = MRI.getVRegDef(Ptr2Reg);
1033 Offset = Imm / Size;
1038 // If we haven't folded anything into our addressing mode yet, try to fold
1039 // a frame index into the base+offset.
1040 if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1041 I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());
1043 I.addOperand(MachineOperand::CreateImm(Offset));
1045 // If we're storing a 0, use WZR/XZR.
1046 if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
1047 if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
1048 if (I.getOpcode() == AArch64::STRWui)
1049 I.getOperand(0).setReg(AArch64::WZR);
1050 else if (I.getOpcode() == AArch64::STRXui)
1051 I.getOperand(0).setReg(AArch64::XZR);
1055 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1058 case TargetOpcode::G_SMULH:
1059 case TargetOpcode::G_UMULH: {
1060 // Reject the various things we don't support yet.
1061 if (unsupportedBinOp(I, RBI, MRI, TRI))
1064 const unsigned DefReg = I.getOperand(0).getReg();
1065 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1067 if (RB.getID() != AArch64::GPRRegBankID) {
1068 DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
1072 if (Ty != LLT::scalar(64)) {
1073 DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
1074 << ", expected: " << LLT::scalar(64) << '\n');
1078 unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
1080 I.setDesc(TII.get(NewOpc));
1082 // Now that we selected an opcode, we need to constrain the register
1083 // operands to use appropriate classes.
1084 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1086 case TargetOpcode::G_FADD:
1087 case TargetOpcode::G_FSUB:
1088 case TargetOpcode::G_FMUL:
1089 case TargetOpcode::G_FDIV:
1091 case TargetOpcode::G_OR:
1092 case TargetOpcode::G_SHL:
1093 case TargetOpcode::G_LSHR:
1094 case TargetOpcode::G_ASHR:
1095 case TargetOpcode::G_GEP: {
1096 // Reject the various things we don't support yet.
1097 if (unsupportedBinOp(I, RBI, MRI, TRI))
1100 const unsigned OpSize = Ty.getSizeInBits();
1102 const unsigned DefReg = I.getOperand(0).getReg();
1103 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1105 const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
1106 if (NewOpc == I.getOpcode())
1109 I.setDesc(TII.get(NewOpc));
1110 // FIXME: Should the type be always reset in setDesc?
1112 // Now that we selected an opcode, we need to constrain the register
1113 // operands to use appropriate classes.
1114 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1117 case TargetOpcode::G_PTR_MASK: {
1118 uint64_t Align = I.getOperand(2).getImm();
1119 if (Align >= 64 || Align == 0)
1122 uint64_t Mask = ~((1ULL << Align) - 1);
1123 I.setDesc(TII.get(AArch64::ANDXri));
1124 I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));
1126 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1128 case TargetOpcode::G_PTRTOINT:
1129 case TargetOpcode::G_TRUNC: {
1130 const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
1131 const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
1133 const unsigned DstReg = I.getOperand(0).getReg();
1134 const unsigned SrcReg = I.getOperand(1).getReg();
1136 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1137 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
1139 if (DstRB.getID() != SrcRB.getID()) {
1140 DEBUG(dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
1144 if (DstRB.getID() == AArch64::GPRRegBankID) {
1145 const TargetRegisterClass *DstRC =
1146 getRegClassForTypeOnBank(DstTy, DstRB, RBI);
1150 const TargetRegisterClass *SrcRC =
1151 getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
1155 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1156 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1157 DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
1161 if (DstRC == SrcRC) {
1162 // Nothing to be done
1163 } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
1164 SrcTy == LLT::scalar(64)) {
1165 llvm_unreachable("TableGen can import this case");
1167 } else if (DstRC == &AArch64::GPR32RegClass &&
1168 SrcRC == &AArch64::GPR64RegClass) {
1169 I.getOperand(1).setSubReg(AArch64::sub_32);
1171 DEBUG(dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
1175 I.setDesc(TII.get(TargetOpcode::COPY));
1177 } else if (DstRB.getID() == AArch64::FPRRegBankID) {
1178 if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
1179 I.setDesc(TII.get(AArch64::XTNv4i16));
1180 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1188 case TargetOpcode::G_ANYEXT: {
1189 const unsigned DstReg = I.getOperand(0).getReg();
1190 const unsigned SrcReg = I.getOperand(1).getReg();
1192 const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
1193 if (RBDst.getID() != AArch64::GPRRegBankID) {
1194 DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n");
1198 const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
1199 if (RBSrc.getID() != AArch64::GPRRegBankID) {
1200 DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n");
1204 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
1207 DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
1211 if (DstSize != 64 && DstSize > 32) {
1212 DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
1213 << ", expected: 32 or 64\n");
1216 // At this point G_ANYEXT is just like a plain COPY, but we need
1217 // to explicitly form the 64-bit value if any.
1219 unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
1220 BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
1224 .addImm(AArch64::sub_32);
1225 I.getOperand(1).setReg(ExtSrc);
1227 return selectCopy(I, TII, MRI, TRI, RBI);
1230 case TargetOpcode::G_ZEXT:
1231 case TargetOpcode::G_SEXT: {
1232 unsigned Opcode = I.getOpcode();
1233 const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
1234 SrcTy = MRI.getType(I.getOperand(1).getReg());
1235 const bool isSigned = Opcode == TargetOpcode::G_SEXT;
1236 const unsigned DefReg = I.getOperand(0).getReg();
1237 const unsigned SrcReg = I.getOperand(1).getReg();
1238 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
1240 if (RB.getID() != AArch64::GPRRegBankID) {
1241 DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
1242 << ", expected: GPR\n");
1247 if (DstTy == LLT::scalar(64)) {
1248 // FIXME: Can we avoid manually doing this?
1249 if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
1250 DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
1255 const unsigned SrcXReg =
1256 MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1257 BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
1261 .addImm(AArch64::sub_32);
1263 const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
1264 ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
1268 .addImm(SrcTy.getSizeInBits() - 1);
1269 } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
1270 const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
1271 ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
1275 .addImm(SrcTy.getSizeInBits() - 1);
1280 constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
1282 I.eraseFromParent();
1286 case TargetOpcode::G_SITOFP:
1287 case TargetOpcode::G_UITOFP:
1288 case TargetOpcode::G_FPTOSI:
1289 case TargetOpcode::G_FPTOUI: {
1290 const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
1291 SrcTy = MRI.getType(I.getOperand(1).getReg());
1292 const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
1293 if (NewOpc == Opcode)
1296 I.setDesc(TII.get(NewOpc));
1297 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1303 case TargetOpcode::G_INTTOPTR:
1304 // The importer is currently unable to import pointer types since they
1305 // didn't exist in SelectionDAG.
1306 return selectCopy(I, TII, MRI, TRI, RBI);
1308 case TargetOpcode::G_BITCAST:
1309 // Imported SelectionDAG rules can handle every bitcast except those that
1310 // bitcast from a type to the same type. Ideally, these shouldn't occur
1311 // but we might not run an optimizer that deletes them.
1312 if (MRI.getType(I.getOperand(0).getReg()) ==
1313 MRI.getType(I.getOperand(1).getReg()))
1314 return selectCopy(I, TII, MRI, TRI, RBI);
1317 case TargetOpcode::G_SELECT: {
1318 if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
1319 DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
1320 << ", expected: " << LLT::scalar(1) << '\n');
1324 const unsigned CondReg = I.getOperand(1).getReg();
1325 const unsigned TReg = I.getOperand(2).getReg();
1326 const unsigned FReg = I.getOperand(3).getReg();
1328 unsigned CSelOpc = 0;
1330 if (Ty == LLT::scalar(32)) {
1331 CSelOpc = AArch64::CSELWr;
1332 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
1333 CSelOpc = AArch64::CSELXr;
1338 MachineInstr &TstMI =
1339 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
1340 .addDef(AArch64::WZR)
1342 .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
1344 MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
1345 .addDef(I.getOperand(0).getReg())
1348 .addImm(AArch64CC::NE);
1350 constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
1351 constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
1353 I.eraseFromParent();
1356 case TargetOpcode::G_ICMP: {
1357 if (Ty != LLT::scalar(32)) {
1358 DEBUG(dbgs() << "G_ICMP result has type: " << Ty
1359 << ", expected: " << LLT::scalar(32) << '\n');
1363 unsigned CmpOpc = 0;
1366 LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1367 if (CmpTy == LLT::scalar(32)) {
1368 CmpOpc = AArch64::SUBSWrr;
1369 ZReg = AArch64::WZR;
1370 } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
1371 CmpOpc = AArch64::SUBSXrr;
1372 ZReg = AArch64::XZR;
1377 // CSINC increments the result by one when the condition code is false.
1378 // Therefore, we have to invert the predicate to get an increment by 1 when
1379 // the predicate is true.
1380 const AArch64CC::CondCode invCC =
1381 changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
1382 (CmpInst::Predicate)I.getOperand(1).getPredicate()));
1384 MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1386 .addUse(I.getOperand(2).getReg())
1387 .addUse(I.getOperand(3).getReg());
1389 MachineInstr &CSetMI =
1390 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1391 .addDef(I.getOperand(0).getReg())
1392 .addUse(AArch64::WZR)
1393 .addUse(AArch64::WZR)
1396 constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1397 constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1399 I.eraseFromParent();
1403 case TargetOpcode::G_FCMP: {
1404 if (Ty != LLT::scalar(32)) {
1405 DEBUG(dbgs() << "G_FCMP result has type: " << Ty
1406 << ", expected: " << LLT::scalar(32) << '\n');
1410 unsigned CmpOpc = 0;
1411 LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
1412 if (CmpTy == LLT::scalar(32)) {
1413 CmpOpc = AArch64::FCMPSrr;
1414 } else if (CmpTy == LLT::scalar(64)) {
1415 CmpOpc = AArch64::FCMPDrr;
1422 AArch64CC::CondCode CC1, CC2;
1423 changeFCMPPredToAArch64CC(
1424 (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
1426 MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
1427 .addUse(I.getOperand(2).getReg())
1428 .addUse(I.getOperand(3).getReg());
1430 const unsigned DefReg = I.getOperand(0).getReg();
1431 unsigned Def1Reg = DefReg;
1432 if (CC2 != AArch64CC::AL)
1433 Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1435 MachineInstr &CSetMI =
1436 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1438 .addUse(AArch64::WZR)
1439 .addUse(AArch64::WZR)
1440 .addImm(getInvertedCondCode(CC1));
1442 if (CC2 != AArch64CC::AL) {
1443 unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
1444 MachineInstr &CSet2MI =
1445 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
1447 .addUse(AArch64::WZR)
1448 .addUse(AArch64::WZR)
1449 .addImm(getInvertedCondCode(CC2));
1450 MachineInstr &OrMI =
1451 *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
1455 constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
1456 constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
1459 constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
1460 constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
1462 I.eraseFromParent();
1465 case TargetOpcode::G_VASTART:
1466 return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
1467 : selectVaStartAAPCS(I, MF, MRI);
1468 case TargetOpcode::G_IMPLICIT_DEF:
1469 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1476 /// SelectArithImmed - Select an immediate value that can be represented as
1477 /// a 12-bit value shifted left by either 0 or 12. If so, return true with
1478 /// Val set to the 12-bit value and Shift set to the shifter operand.
1479 InstructionSelector::ComplexRendererFns
1480 AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
1481 MachineInstr &MI = *Root.getParent();
1482 MachineBasicBlock &MBB = *MI.getParent();
1483 MachineFunction &MF = *MBB.getParent();
1484 MachineRegisterInfo &MRI = MF.getRegInfo();
1486 // This function is called from the addsub_shifted_imm ComplexPattern,
1487 // which lists [imm] as the list of opcode it's interested in, however
1488 // we still need to check whether the operand is actually an immediate
1489 // here because the ComplexPattern opcode list is only used in
1490 // root-level opcode matching.
1493 Immed = Root.getImm();
1494 else if (Root.isCImm())
1495 Immed = Root.getCImm()->getZExtValue();
1496 else if (Root.isReg()) {
1497 MachineInstr *Def = MRI.getVRegDef(Root.getReg());
1498 if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
1500 MachineOperand &Op1 = Def->getOperand(1);
1501 if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
1503 Immed = Op1.getCImm()->getZExtValue();
1509 if (Immed >> 12 == 0) {
1511 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
1513 Immed = Immed >> 12;
1517 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
1519 [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
1520 [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
1524 /// Select a "register plus unscaled signed 9-bit immediate" address. This
1525 /// should only match when there is an offset that is not valid for a scaled
1526 /// immediate addressing mode. The "Size" argument is the size in bytes of the
1527 /// memory reference, which is needed here to know what is valid for a scaled
1529 InstructionSelector::ComplexRendererFns
1530 AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
1531 unsigned Size) const {
1532 MachineRegisterInfo &MRI =
1533 Root.getParent()->getParent()->getParent()->getRegInfo();
1538 if (!isBaseWithConstantOffset(Root, MRI))
1541 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
1545 MachineOperand &OffImm = RootDef->getOperand(2);
1546 if (!OffImm.isReg())
1548 MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
1549 if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
1552 MachineOperand &RHSOp1 = RHS->getOperand(1);
1553 if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
1555 RHSC = RHSOp1.getCImm()->getSExtValue();
1557 // If the offset is valid as a scaled immediate, don't match here.
1558 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
1560 if (RHSC >= -256 && RHSC < 256) {
1561 MachineOperand &Base = RootDef->getOperand(1);
1563 [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
1564 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
1570 /// Select a "register plus scaled unsigned 12-bit immediate" address. The
1571 /// "Size" argument is the size in bytes of the memory reference, which
1572 /// determines the scale.
1573 InstructionSelector::ComplexRendererFns
1574 AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
1575 unsigned Size) const {
1576 MachineRegisterInfo &MRI =
1577 Root.getParent()->getParent()->getParent()->getRegInfo();
1582 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
1586 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
1588 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
1589 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
1593 if (isBaseWithConstantOffset(Root, MRI)) {
1594 MachineOperand &LHS = RootDef->getOperand(1);
1595 MachineOperand &RHS = RootDef->getOperand(2);
1596 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
1597 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
1598 if (LHSDef && RHSDef) {
1599 int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
1600 unsigned Scale = Log2_32(Size);
1601 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
1602 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1604 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
1605 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
1609 [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
1610 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
1616 // Before falling back to our general case, check if the unscaled
1617 // instructions can handle this. If so, that's preferable.
1618 if (selectAddrModeUnscaled(Root, Size).hasValue())
1622 [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
1623 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
1628 InstructionSelector *
1629 createAArch64InstructionSelector(const AArch64TargetMachine &TM,
1630 AArch64Subtarget &Subtarget,
1631 AArch64RegisterBankInfo &RBI) {
1632 return new AArch64InstructionSelector(TM, Subtarget, RBI);