contrib/llvm-project/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

   1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a printer that converts from our internal representation
  10 // of machine-dependent LLVM code to the AArch64 assembly language.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64.h"
  15 #include "AArch64MCInstLower.h"
  16 #include "AArch64MachineFunctionInfo.h"
  17 #include "AArch64RegisterInfo.h"
  18 #include "AArch64Subtarget.h"
  19 #include "AArch64TargetObjectFile.h"
  20 #include "MCTargetDesc/AArch64AddressingModes.h"
  21 #include "MCTargetDesc/AArch64InstPrinter.h"
  22 #include "MCTargetDesc/AArch64MCExpr.h"
  23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
  24 #include "MCTargetDesc/AArch64TargetStreamer.h"
  25 #include "TargetInfo/AArch64TargetInfo.h"
  26 #include "Utils/AArch64BaseInfo.h"
  27 #include "llvm/ADT/SmallString.h"
  28 #include "llvm/ADT/SmallVector.h"
  29 #include "llvm/ADT/StringRef.h"
  30 #include "llvm/ADT/Triple.h"
  31 #include "llvm/ADT/Twine.h"
  32 #include "llvm/BinaryFormat/COFF.h"
  33 #include "llvm/BinaryFormat/ELF.h"
  34 #include "llvm/CodeGen/AsmPrinter.h"
  35 #include "llvm/CodeGen/MachineBasicBlock.h"
  36 #include "llvm/CodeGen/MachineFunction.h"
  37 #include "llvm/CodeGen/MachineInstr.h"
  38 #include "llvm/CodeGen/MachineJumpTableInfo.h"
  39 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
  40 #include "llvm/CodeGen/MachineOperand.h"
  41 #include "llvm/CodeGen/StackMaps.h"
  42 #include "llvm/CodeGen/TargetRegisterInfo.h"
  43 #include "llvm/IR/DataLayout.h"
  44 #include "llvm/IR/DebugInfoMetadata.h"
  45 #include "llvm/MC/MCAsmInfo.h"
  46 #include "llvm/MC/MCContext.h"
  47 #include "llvm/MC/MCInst.h"
  48 #include "llvm/MC/MCInstBuilder.h"
  49 #include "llvm/MC/MCSectionELF.h"
  50 #include "llvm/MC/MCStreamer.h"
  51 #include "llvm/MC/MCSymbol.h"
  52 #include "llvm/Support/Casting.h"
  53 #include "llvm/Support/ErrorHandling.h"
  54 #include "llvm/Support/TargetRegistry.h"
  55 #include "llvm/Support/raw_ostream.h"
  56 #include "llvm/Target/TargetMachine.h"
  57 #include <algorithm>
  58 #include <cassert>
  59 #include <cstdint>
  60 #include <map>
  61 #include <memory>
  62
  63 using namespace llvm;
  64
  65 #define DEBUG_TYPE "asm-printer"
  66
  67 namespace {
  68
  69 class AArch64AsmPrinter : public AsmPrinter {
  70   AArch64MCInstLower MCInstLowering;
  71   StackMaps SM;
  72   const AArch64Subtarget *STI;
  73
  74 public:
  75   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
  76       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
  77         SM(*this) {}
  78
  79   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
  80
  81   /// Wrapper for MCInstLowering.lowerOperand() for the
  82   /// tblgen'erated pseudo lowering.
  83   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
  84     return MCInstLowering.lowerOperand(MO, MCOp);
  85   }
  86
  87   void EmitStartOfAsmFile(Module &M) override;
  88   void EmitJumpTableInfo() override;
  89   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
  90                           const MachineBasicBlock *MBB, unsigned JTI);
  91
  92   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
  93
  94   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
  95                      const MachineInstr &MI);
  96   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  97                        const MachineInstr &MI);
  98
  99   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
 100   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
 101   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
 102
 103   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
 104   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
 105   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
 106   void EmitHwasanMemaccessSymbols(Module &M);
 107
 108   void EmitSled(const MachineInstr &MI, SledKind Kind);
 109
 110   /// tblgen'erated driver function for lowering simple MI->MC
 111   /// pseudo instructions.
 112   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
 113                                    const MachineInstr *MI);
 114
 115   void EmitInstruction(const MachineInstr *MI) override;
 116
 117   void getAnalysisUsage(AnalysisUsage &AU) const override {
 118     AsmPrinter::getAnalysisUsage(AU);
 119     AU.setPreservesAll();
 120   }
 121
 122   bool runOnMachineFunction(MachineFunction &MF) override {
 123     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
 124     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
 125
 126     SetupMachineFunction(MF);
 127
 128     if (STI->isTargetCOFF()) {
 129       bool Internal = MF.getFunction().hasInternalLinkage();
 130       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
 131                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
 132       int Type =
 133         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
 134
 135       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
 136       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
 137       OutStreamer->EmitCOFFSymbolType(Type);
 138       OutStreamer->EndCOFFSymbolDef();
 139     }
 140
 141     // Emit the rest of the function body.
 142     EmitFunctionBody();
 143
 144     // Emit the XRay table for this function.
 145     emitXRayTable();
 146
 147     // We didn't modify anything.
 148     return false;
 149   }
 150
 151 private:
 152   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
 153   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
 154   bool printAsmRegInClass(const MachineOperand &MO,
 155                           const TargetRegisterClass *RC, unsigned AltName,
 156                           raw_ostream &O);
 157
 158   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 159                        const char *ExtraCode, raw_ostream &O) override;
 160   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
 161                              const char *ExtraCode, raw_ostream &O) override;
 162
 163   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
 164
 165   void EmitFunctionBodyEnd() override;
 166
 167   MCSymbol *GetCPISymbol(unsigned CPID) const override;
 168   void EmitEndOfAsmFile(Module &M) override;
 169
 170   AArch64FunctionInfo *AArch64FI = nullptr;
 171
 172   /// Emit the LOHs contained in AArch64FI.
 173   void EmitLOHs();
 174
 175   /// Emit instruction to set float register to zero.
 176   void EmitFMov0(const MachineInstr &MI);
 177
 178   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
 179
 180   MInstToMCSymbol LOHInstToLabel;
 181 };
 182
 183 } // end anonymous namespace
 184
 185 void AArch64AsmPrinter::EmitStartOfAsmFile(Module &M) {
 186   if (!TM.getTargetTriple().isOSBinFormatELF())
 187     return;
 188
 189   // Assemble feature flags that may require creation of a note section.
 190   unsigned Flags = ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
 191                    ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
 192
 193   if (any_of(M, [](const Function &F) {
 194         return !F.isDeclaration() &&
 195                !F.hasFnAttribute("branch-target-enforcement");
 196       })) {
 197     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
 198   }
 199
 200   if ((Flags & ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI) == 0 &&
 201       any_of(M, [](const Function &F) {
 202         return F.hasFnAttribute("branch-target-enforcement");
 203       })) {
 204     errs() << "warning: some functions compiled with BTI and some compiled "
 205               "without BTI\n"
 206            << "warning: not setting BTI in feature flags\n";
 207   }
 208
 209   if (any_of(M, [](const Function &F) {
 210         if (F.isDeclaration())
 211           return false;
 212         Attribute A = F.getFnAttribute("sign-return-address");
 213         return !A.isStringAttribute() || A.getValueAsString() == "none";
 214       })) {
 215     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
 216   }
 217
 218   if (Flags == 0)
 219     return;
 220
 221   // Emit a .note.gnu.property section with the flags.
 222   MCSection *Cur = OutStreamer->getCurrentSectionOnly();
 223   MCSection *Nt = MMI->getContext().getELFSection(
 224       ".note.gnu.property", ELF::SHT_NOTE, ELF::SHF_ALLOC);
 225   OutStreamer->SwitchSection(Nt);
 226
 227   // Emit the note header.
 228   EmitAlignment(Align(8));
 229   OutStreamer->EmitIntValue(4, 4);     // data size for "GNU\0"
 230   OutStreamer->EmitIntValue(4 * 4, 4); // Elf_Prop size
 231   OutStreamer->EmitIntValue(ELF::NT_GNU_PROPERTY_TYPE_0, 4);
 232   OutStreamer->EmitBytes(StringRef("GNU", 4)); // note name
 233
 234   // Emit the PAC/BTI properties.
 235   OutStreamer->EmitIntValue(ELF::GNU_PROPERTY_AARCH64_FEATURE_1_AND, 4);
 236   OutStreamer->EmitIntValue(4, 4);     // data size
 237   OutStreamer->EmitIntValue(Flags, 4); // data
 238   OutStreamer->EmitIntValue(0, 4);     // pad
 239
 240   OutStreamer->endSection(Nt);
 241   OutStreamer->SwitchSection(Cur);
 242 }
 243
 244 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
 245 {
 246   const Function &F = MF->getFunction();
 247   if (F.hasFnAttribute("patchable-function-entry")) {
 248     unsigned Num;
 249     if (F.getFnAttribute("patchable-function-entry")
 250             .getValueAsString()
 251             .getAsInteger(10, Num))
 252       return;
 253     emitNops(Num);
 254     return;
 255   }
 256
 257   EmitSled(MI, SledKind::FUNCTION_ENTER);
 258 }
 259
 260 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
 261 {
 262   EmitSled(MI, SledKind::FUNCTION_EXIT);
 263 }
 264
 265 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
 266 {
 267   EmitSled(MI, SledKind::TAIL_CALL);
 268 }
 269
 270 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
 271 {
 272   static const int8_t NoopsInSledCount = 7;
 273   // We want to emit the following pattern:
 274   //
 275   // .Lxray_sled_N:
 276   //   ALIGN
 277   //   B #32
 278   //   ; 7 NOP instructions (28 bytes)
 279   // .tmpN
 280   //
 281   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
 282   // over the full 32 bytes (8 instructions) with the following pattern:
 283   //
 284   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
 285   //   LDR W0, #12 ; W0 := function ID
 286   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
 287   //   BLR X16 ; call the tracing trampoline
 288   //   ;DATA: 32 bits of function ID
 289   //   ;DATA: lower 32 bits of the address of the trampoline
 290   //   ;DATA: higher 32 bits of the address of the trampoline
 291   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
 292   //
 293   OutStreamer->EmitCodeAlignment(4);
 294   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
 295   OutStreamer->EmitLabel(CurSled);
 296   auto Target = OutContext.createTempSymbol();
 297
 298   // Emit "B #32" instruction, which jumps over the next 28 bytes.
 299   // The operand has to be the number of 4-byte instructions to jump over,
 300   // including the current instruction.
 301   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
 302
 303   for (int8_t I = 0; I < NoopsInSledCount; I++)
 304     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 305
 306   OutStreamer->EmitLabel(Target);
 307   recordSled(CurSled, MI, Kind);
 308 }
 309
 310 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
 311   Register Reg = MI.getOperand(0).getReg();
 312   bool IsShort =
 313       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
 314   uint32_t AccessInfo = MI.getOperand(1).getImm();
 315   MCSymbol *&Sym =
 316       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
 317   if (!Sym) {
 318     // FIXME: Make this work on non-ELF.
 319     if (!TM.getTargetTriple().isOSBinFormatELF())
 320       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
 321
 322     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
 323                           utostr(AccessInfo);
 324     if (IsShort)
 325       SymName += "_short";
 326     Sym = OutContext.getOrCreateSymbol(SymName);
 327   }
 328
 329   EmitToStreamer(*OutStreamer,
 330                  MCInstBuilder(AArch64::BL)
 331                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
 332 }
 333
 334 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
 335   if (HwasanMemaccessSymbols.empty())
 336     return;
 337
 338   const Triple &TT = TM.getTargetTriple();
 339   assert(TT.isOSBinFormatELF());
 340   std::unique_ptr<MCSubtargetInfo> STI(
 341       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
 342
 343   MCSymbol *HwasanTagMismatchV1Sym =
 344       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
 345   MCSymbol *HwasanTagMismatchV2Sym =
 346       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
 347
 348   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
 349       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
 350   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
 351       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
 352
 353   for (auto &P : HwasanMemaccessSymbols) {
 354     unsigned Reg = std::get<0>(P.first);
 355     bool IsShort = std::get<1>(P.first);
 356     uint32_t AccessInfo = std::get<2>(P.first);
 357     const MCSymbolRefExpr *HwasanTagMismatchRef =
 358         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
 359     MCSymbol *Sym = P.second;
 360
 361     OutStreamer->SwitchSection(OutContext.getELFSection(
 362         ".text.hot", ELF::SHT_PROGBITS,
 363         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
 364         Sym->getName()));
 365
 366     OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
 367     OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak);
 368     OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden);
 369     OutStreamer->EmitLabel(Sym);
 370
 371     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
 372                                      .addReg(AArch64::X16)
 373                                      .addReg(Reg)
 374                                      .addImm(4)
 375                                      .addImm(55),
 376                                  *STI);
 377     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX)
 378                                      .addReg(AArch64::W16)
 379                                      .addReg(AArch64::X9)
 380                                      .addReg(AArch64::X16)
 381                                      .addImm(0)
 382                                      .addImm(0),
 383                                  *STI);
 384     OutStreamer->EmitInstruction(
 385         MCInstBuilder(AArch64::SUBSXrs)
 386             .addReg(AArch64::XZR)
 387             .addReg(AArch64::X16)
 388             .addReg(Reg)
 389             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
 390         *STI);
 391     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
 392     OutStreamer->EmitInstruction(
 393         MCInstBuilder(AArch64::Bcc)
 394             .addImm(AArch64CC::NE)
 395             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
 396                                              OutContext)),
 397         *STI);
 398     MCSymbol *ReturnSym = OutContext.createTempSymbol();
 399     OutStreamer->EmitLabel(ReturnSym);
 400     OutStreamer->EmitInstruction(
 401         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
 402     OutStreamer->EmitLabel(HandleMismatchOrPartialSym);
 403
 404     if (IsShort) {
 405       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri)
 406                                        .addReg(AArch64::WZR)
 407                                        .addReg(AArch64::W16)
 408                                        .addImm(15)
 409                                        .addImm(0),
 410                                    *STI);
 411       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
 412       OutStreamer->EmitInstruction(
 413           MCInstBuilder(AArch64::Bcc)
 414               .addImm(AArch64CC::HI)
 415               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 416           *STI);
 417
 418       OutStreamer->EmitInstruction(
 419           MCInstBuilder(AArch64::ANDXri)
 420               .addReg(AArch64::X17)
 421               .addReg(Reg)
 422               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
 423           *STI);
 424       unsigned Size = 1 << (AccessInfo & 0xf);
 425       if (Size != 1)
 426         OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri)
 427                                          .addReg(AArch64::X17)
 428                                          .addReg(AArch64::X17)
 429                                          .addImm(Size - 1)
 430                                          .addImm(0),
 431                                      *STI);
 432       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
 433                                        .addReg(AArch64::WZR)
 434                                        .addReg(AArch64::W16)
 435                                        .addReg(AArch64::W17)
 436                                        .addImm(0),
 437                                    *STI);
 438       OutStreamer->EmitInstruction(
 439           MCInstBuilder(AArch64::Bcc)
 440               .addImm(AArch64CC::LS)
 441               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 442           *STI);
 443
 444       OutStreamer->EmitInstruction(
 445           MCInstBuilder(AArch64::ORRXri)
 446               .addReg(AArch64::X16)
 447               .addReg(Reg)
 448               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
 449           *STI);
 450       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui)
 451                                        .addReg(AArch64::W16)
 452                                        .addReg(AArch64::X16)
 453                                        .addImm(0),
 454                                    *STI);
 455       OutStreamer->EmitInstruction(
 456           MCInstBuilder(AArch64::SUBSXrs)
 457               .addReg(AArch64::XZR)
 458               .addReg(AArch64::X16)
 459               .addReg(Reg)
 460               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
 461           *STI);
 462       OutStreamer->EmitInstruction(
 463           MCInstBuilder(AArch64::Bcc)
 464               .addImm(AArch64CC::EQ)
 465               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
 466           *STI);
 467
 468       OutStreamer->EmitLabel(HandleMismatchSym);
 469     }
 470
 471     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre)
 472                                      .addReg(AArch64::SP)
 473                                      .addReg(AArch64::X0)
 474                                      .addReg(AArch64::X1)
 475                                      .addReg(AArch64::SP)
 476                                      .addImm(-32),
 477                                  *STI);
 478     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXi)
 479                                      .addReg(AArch64::FP)
 480                                      .addReg(AArch64::LR)
 481                                      .addReg(AArch64::SP)
 482                                      .addImm(29),
 483                                  *STI);
 484
 485     if (Reg != AArch64::X0)
 486       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs)
 487                                        .addReg(AArch64::X0)
 488                                        .addReg(AArch64::XZR)
 489                                        .addReg(Reg)
 490                                        .addImm(0),
 491                                    *STI);
 492     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi)
 493                                      .addReg(AArch64::X1)
 494                                      .addImm(AccessInfo)
 495                                      .addImm(0),
 496                                  *STI);
 497
 498     // Intentionally load the GOT entry and branch to it, rather than possibly
 499     // late binding the function, which may clobber the registers before we have
 500     // a chance to save them.
 501     OutStreamer->EmitInstruction(
 502         MCInstBuilder(AArch64::ADRP)
 503             .addReg(AArch64::X16)
 504             .addExpr(AArch64MCExpr::create(
 505                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
 506                 OutContext)),
 507         *STI);
 508     OutStreamer->EmitInstruction(
 509         MCInstBuilder(AArch64::LDRXui)
 510             .addReg(AArch64::X16)
 511             .addReg(AArch64::X16)
 512             .addExpr(AArch64MCExpr::create(
 513                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
 514                 OutContext)),
 515         *STI);
 516     OutStreamer->EmitInstruction(
 517         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
 518   }
 519 }
 520
 521 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
 522   EmitHwasanMemaccessSymbols(M);
 523
 524   const Triple &TT = TM.getTargetTriple();
 525   if (TT.isOSBinFormatMachO()) {
 526     // Funny Darwin hack: This flag tells the linker that no global symbols
 527     // contain code that falls through to other global symbols (e.g. the obvious
 528     // implementation of multiple entry points).  If this doesn't occur, the
 529     // linker can safely perform dead code stripping.  Since LLVM never
 530     // generates code that does this, it is always safe to set.
 531     OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
 532   }
 533   emitStackMaps(SM);
 534 }
 535
 536 void AArch64AsmPrinter::EmitLOHs() {
 537   SmallVector<MCSymbol *, 3> MCArgs;
 538
 539   for (const auto &D : AArch64FI->getLOHContainer()) {
 540     for (const MachineInstr *MI : D.getArgs()) {
 541       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
 542       assert(LabelIt != LOHInstToLabel.end() &&
 543              "Label hasn't been inserted for LOH related instruction");
 544       MCArgs.push_back(LabelIt->second);
 545     }
 546     OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
 547     MCArgs.clear();
 548   }
 549 }
 550
 551 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
 552   if (!AArch64FI->getLOHRelated().empty())
 553     EmitLOHs();
 554 }
 555
 556 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
 557 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
 558   // Darwin uses a linker-private symbol name for constant-pools (to
 559   // avoid addends on the relocation?), ELF has no such concept and
 560   // uses a normal private symbol.
 561   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
 562     return OutContext.getOrCreateSymbol(
 563         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
 564         Twine(getFunctionNumber()) + "_" + Twine(CPID));
 565
 566   return AsmPrinter::GetCPISymbol(CPID);
 567 }
 568
 569 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
 570                                      raw_ostream &O) {
 571   const MachineOperand &MO = MI->getOperand(OpNum);
 572   switch (MO.getType()) {
 573   default:
 574     llvm_unreachable("<unknown operand type>");
 575   case MachineOperand::MO_Register: {
 576     Register Reg = MO.getReg();
 577     assert(Register::isPhysicalRegister(Reg));
 578     assert(!MO.getSubReg() && "Subregs should be eliminated!");
 579     O << AArch64InstPrinter::getRegisterName(Reg);
 580     break;
 581   }
 582   case MachineOperand::MO_Immediate: {
 583     O << MO.getImm();
 584     break;
 585   }
 586   case MachineOperand::MO_GlobalAddress: {
 587     PrintSymbolOperand(MO, O);
 588     break;
 589   }
 590   case MachineOperand::MO_BlockAddress: {
 591     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
 592     Sym->print(O, MAI);
 593     break;
 594   }
 595   }
 596 }
 597
 598 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
 599                                           raw_ostream &O) {
 600   Register Reg = MO.getReg();
 601   switch (Mode) {
 602   default:
 603     return true; // Unknown mode.
 604   case 'w':
 605     Reg = getWRegFromXReg(Reg);
 606     break;
 607   case 'x':
 608     Reg = getXRegFromWReg(Reg);
 609     break;
 610   }
 611
 612   O << AArch64InstPrinter::getRegisterName(Reg);
 613   return false;
 614 }
 615
 616 // Prints the register in MO using class RC using the offset in the
 617 // new register class. This should not be used for cross class
 618 // printing.
 619 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
 620                                            const TargetRegisterClass *RC,
 621                                            unsigned AltName, raw_ostream &O) {
 622   assert(MO.isReg() && "Should only get here with a register!");
 623   const TargetRegisterInfo *RI = STI->getRegisterInfo();
 624   Register Reg = MO.getReg();
 625   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
 626   assert(RI->regsOverlap(RegToPrint, Reg));
 627   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
 628   return false;
 629 }
 630
 631 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 632                                         const char *ExtraCode, raw_ostream &O) {
 633   const MachineOperand &MO = MI->getOperand(OpNum);
 634
 635   // First try the generic code, which knows about modifiers like 'c' and 'n'.
 636   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
 637     return false;
 638
 639   // Does this asm operand have a single letter operand modifier?
 640   if (ExtraCode && ExtraCode[0]) {
 641     if (ExtraCode[1] != 0)
 642       return true; // Unknown modifier.
 643
 644     switch (ExtraCode[0]) {
 645     default:
 646       return true; // Unknown modifier.
 647     case 'w':      // Print W register
 648     case 'x':      // Print X register
 649       if (MO.isReg())
 650         return printAsmMRegister(MO, ExtraCode[0], O);
 651       if (MO.isImm() && MO.getImm() == 0) {
 652         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
 653         O << AArch64InstPrinter::getRegisterName(Reg);
 654         return false;
 655       }
 656       printOperand(MI, OpNum, O);
 657       return false;
 658     case 'b': // Print B register.
 659     case 'h': // Print H register.
 660     case 's': // Print S register.
 661     case 'd': // Print D register.
 662     case 'q': // Print Q register.
 663     case 'z': // Print Z register.
 664       if (MO.isReg()) {
 665         const TargetRegisterClass *RC;
 666         switch (ExtraCode[0]) {
 667         case 'b':
 668           RC = &AArch64::FPR8RegClass;
 669           break;
 670         case 'h':
 671           RC = &AArch64::FPR16RegClass;
 672           break;
 673         case 's':
 674           RC = &AArch64::FPR32RegClass;
 675           break;
 676         case 'd':
 677           RC = &AArch64::FPR64RegClass;
 678           break;
 679         case 'q':
 680           RC = &AArch64::FPR128RegClass;
 681           break;
 682         case 'z':
 683           RC = &AArch64::ZPRRegClass;
 684           break;
 685         default:
 686           return true;
 687         }
 688         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
 689       }
 690       printOperand(MI, OpNum, O);
 691       return false;
 692     }
 693   }
 694
 695   // According to ARM, we should emit x and v registers unless we have a
 696   // modifier.
 697   if (MO.isReg()) {
 698     Register Reg = MO.getReg();
 699
 700     // If this is a w or x register, print an x register.
 701     if (AArch64::GPR32allRegClass.contains(Reg) ||
 702         AArch64::GPR64allRegClass.contains(Reg))
 703       return printAsmMRegister(MO, 'x', O);
 704
 705     unsigned AltName = AArch64::NoRegAltName;
 706     const TargetRegisterClass *RegClass;
 707     if (AArch64::ZPRRegClass.contains(Reg)) {
 708       RegClass = &AArch64::ZPRRegClass;
 709     } else if (AArch64::PPRRegClass.contains(Reg)) {
 710       RegClass = &AArch64::PPRRegClass;
 711     } else {
 712       RegClass = &AArch64::FPR128RegClass;
 713       AltName = AArch64::vreg;
 714     }
 715
 716     // If this is a b, h, s, d, or q register, print it as a v register.
 717     return printAsmRegInClass(MO, RegClass, AltName, O);
 718   }
 719
 720   printOperand(MI, OpNum, O);
 721   return false;
 722 }
 723
 724 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 725                                               unsigned OpNum,
 726                                               const char *ExtraCode,
 727                                               raw_ostream &O) {
 728   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
 729     return true; // Unknown modifier.
 730
 731   const MachineOperand &MO = MI->getOperand(OpNum);
 732   assert(MO.isReg() && "unexpected inline asm memory operand");
 733   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
 734   return false;
 735 }
 736
 737 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
 738                                                raw_ostream &OS) {
 739   unsigned NOps = MI->getNumOperands();
 740   assert(NOps == 4);
 741   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
 742   // cast away const; DIetc do not take const operands for some reason.
 743   OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
 744             ->getName();
 745   OS << " <- ";
 746   // Frame address.  Currently handles register +- offset only.
 747   assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
 748   OS << '[';
 749   printOperand(MI, 0, OS);
 750   OS << '+';
 751   printOperand(MI, 1, OS);
 752   OS << ']';
 753   OS << "+";
 754   printOperand(MI, NOps - 2, OS);
 755 }
 756
 757 void AArch64AsmPrinter::EmitJumpTableInfo() {
 758   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
 759   if (!MJTI) return;
 760
 761   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
 762   if (JT.empty()) return;
 763
 764   const Function &F = MF->getFunction();
 765   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
 766   bool JTInDiffSection =
 767       !STI->isTargetCOFF() ||
 768       !TLOF.shouldPutJumpTableInFunctionSection(
 769           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
 770           F);
 771   if (JTInDiffSection) {
 772       // Drop it in the readonly section.
 773       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
 774       OutStreamer->SwitchSection(ReadOnlySec);
 775   }
 776
 777   auto AFI = MF->getInfo<AArch64FunctionInfo>();
 778   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
 779     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
 780
 781     // If this jump table was deleted, ignore it.
 782     if (JTBBs.empty()) continue;
 783
 784     unsigned Size = AFI->getJumpTableEntrySize(JTI);
 785     EmitAlignment(Align(Size));
 786     OutStreamer->EmitLabel(GetJTISymbol(JTI));
 787
 788     for (auto *JTBB : JTBBs)
 789       emitJumpTableEntry(MJTI, JTBB, JTI);
 790   }
 791 }
 792
 793 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
 794                                            const MachineBasicBlock *MBB,
 795                                            unsigned JTI) {
 796   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
 797   auto AFI = MF->getInfo<AArch64FunctionInfo>();
 798   unsigned Size = AFI->getJumpTableEntrySize(JTI);
 799
 800   if (Size == 4) {
 801     // .word LBB - LJTI
 802     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
 803     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
 804     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
 805   } else {
 806     // .byte (LBB - LBB) >> 2 (or .hword)
 807     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
 808     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
 809     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
 810     Value = MCBinaryExpr::createLShr(
 811         Value, MCConstantExpr::create(2, OutContext), OutContext);
 812   }
 813
 814   OutStreamer->EmitValue(Value, Size);
 815 }
 816
 817 /// Small jump tables contain an unsigned byte or half, representing the offset
 818 /// from the lowest-addressed possible destination to the desired basic
 819 /// block. Since all instructions are 4-byte aligned, this is further compressed
 820 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
 821 /// materialize the correct destination we need:
 822 ///
 823 ///             adr xDest, .LBB0_0
 824 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
 825 ///             add xDest, xDest, xScratch, lsl #2
 826 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
 827                                                 const llvm::MachineInstr &MI) {
 828   Register DestReg = MI.getOperand(0).getReg();
 829   Register ScratchReg = MI.getOperand(1).getReg();
 830   Register ScratchRegW =
 831       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
 832   Register TableReg = MI.getOperand(2).getReg();
 833   Register EntryReg = MI.getOperand(3).getReg();
 834   int JTIdx = MI.getOperand(4).getIndex();
 835   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
 836
 837   // This has to be first because the compression pass based its reachability
 838   // calculations on the start of the JumpTableDest instruction.
 839   auto Label =
 840       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
 841   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
 842                                   .addReg(DestReg)
 843                                   .addExpr(MCSymbolRefExpr::create(
 844                                       Label, MF->getContext())));
 845
 846   // Load the number of instruction-steps to offset from the label.
 847   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
 848   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
 849                                   .addReg(ScratchRegW)
 850                                   .addReg(TableReg)
 851                                   .addReg(EntryReg)
 852                                   .addImm(0)
 853                                   .addImm(IsByteEntry ? 0 : 1));
 854
 855   // Multiply the steps by 4 and add to the already materialized base label
 856   // address.
 857   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
 858                                   .addReg(DestReg)
 859                                   .addReg(DestReg)
 860                                   .addReg(ScratchReg)
 861                                   .addImm(2));
 862 }
 863
 864 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
 865                                       const MachineInstr &MI) {
 866   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
 867
 868   auto &Ctx = OutStreamer.getContext();
 869   MCSymbol *MILabel = Ctx.createTempSymbol();
 870   OutStreamer.EmitLabel(MILabel);
 871
 872   SM.recordStackMap(*MILabel, MI);
 873   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
 874
 875   // Scan ahead to trim the shadow.
 876   const MachineBasicBlock &MBB = *MI.getParent();
 877   MachineBasicBlock::const_iterator MII(MI);
 878   ++MII;
 879   while (NumNOPBytes > 0) {
 880     if (MII == MBB.end() || MII->isCall() ||
 881         MII->getOpcode() == AArch64::DBG_VALUE ||
 882         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
 883         MII->getOpcode() == TargetOpcode::STACKMAP)
 884       break;
 885     ++MII;
 886     NumNOPBytes -= 4;
 887   }
 888
 889   // Emit nops.
 890   for (unsigned i = 0; i < NumNOPBytes; i += 4)
 891     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 892 }
 893
 894 // Lower a patchpoint of the form:
 895 // [<def>], <id>, <numBytes>, <target>, <numArgs>
 896 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 897                                         const MachineInstr &MI) {
 898   auto &Ctx = OutStreamer.getContext();
 899   MCSymbol *MILabel = Ctx.createTempSymbol();
 900   OutStreamer.EmitLabel(MILabel);
 901   SM.recordPatchPoint(*MILabel, MI);
 902
 903   PatchPointOpers Opers(&MI);
 904
 905   int64_t CallTarget = Opers.getCallTarget().getImm();
 906   unsigned EncodedBytes = 0;
 907   if (CallTarget) {
 908     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
 909            "High 16 bits of call target should be zero.");
 910     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
 911     EncodedBytes = 16;
 912     // Materialize the jump address:
 913     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
 914                                     .addReg(ScratchReg)
 915                                     .addImm((CallTarget >> 32) & 0xFFFF)
 916                                     .addImm(32));
 917     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
 918                                     .addReg(ScratchReg)
 919                                     .addReg(ScratchReg)
 920                                     .addImm((CallTarget >> 16) & 0xFFFF)
 921                                     .addImm(16));
 922     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
 923                                     .addReg(ScratchReg)
 924                                     .addReg(ScratchReg)
 925                                     .addImm(CallTarget & 0xFFFF)
 926                                     .addImm(0));
 927     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
 928   }
 929   // Emit padding.
 930   unsigned NumBytes = Opers.getNumPatchBytes();
 931   assert(NumBytes >= EncodedBytes &&
 932          "Patchpoint can't request size less than the length of a call.");
 933   assert((NumBytes - EncodedBytes) % 4 == 0 &&
 934          "Invalid number of NOP bytes requested!");
 935   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
 936     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 937 }
 938
 939 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
 940   Register DestReg = MI.getOperand(0).getReg();
 941   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
 942     // Convert H/S/D register to corresponding Q register
 943     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
 944       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
 945     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
 946       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
 947     else {
 948       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
 949       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
 950     }
 951     MCInst MOVI;
 952     MOVI.setOpcode(AArch64::MOVIv2d_ns);
 953     MOVI.addOperand(MCOperand::createReg(DestReg));
 954     MOVI.addOperand(MCOperand::createImm(0));
 955     EmitToStreamer(*OutStreamer, MOVI);
 956   } else {
 957     MCInst FMov;
 958     switch (MI.getOpcode()) {
 959     default: llvm_unreachable("Unexpected opcode");
 960     case AArch64::FMOVH0:
 961       FMov.setOpcode(AArch64::FMOVWHr);
 962       FMov.addOperand(MCOperand::createReg(DestReg));
 963       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
 964       break;
 965     case AArch64::FMOVS0:
 966       FMov.setOpcode(AArch64::FMOVWSr);
 967       FMov.addOperand(MCOperand::createReg(DestReg));
 968       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
 969       break;
 970     case AArch64::FMOVD0:
 971       FMov.setOpcode(AArch64::FMOVXDr);
 972       FMov.addOperand(MCOperand::createReg(DestReg));
 973       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
 974       break;
 975     }
 976     EmitToStreamer(*OutStreamer, FMov);
 977   }
 978 }
 979
 980 // Simple pseudo-instructions have their lowering (with expansion to real
 981 // instructions) auto-generated.
 982 #include "AArch64GenMCPseudoLowering.inc"
 983
 984 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
 985   // Do any auto-generated pseudo lowerings.
 986   if (emitPseudoExpansionLowering(*OutStreamer, MI))
 987     return;
 988
 989   if (AArch64FI->getLOHRelated().count(MI)) {
 990     // Generate a label for LOH related instruction
 991     MCSymbol *LOHLabel = createTempSymbol("loh");
 992     // Associate the instruction with the label
 993     LOHInstToLabel[MI] = LOHLabel;
 994     OutStreamer->EmitLabel(LOHLabel);
 995   }
 996
 997   AArch64TargetStreamer *TS =
 998     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
 999   // Do any manual lowerings.
1000   switch (MI->getOpcode()) {
1001   default:
1002     break;
1003     case AArch64::MOVMCSym: {
1004       Register DestReg = MI->getOperand(0).getReg();
1005       const MachineOperand &MO_Sym = MI->getOperand(1);
1006       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1007       MCOperand Hi_MCSym, Lo_MCSym;
1008
1009       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1010       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1011
1012       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1013       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1014
1015       MCInst MovZ;
1016       MovZ.setOpcode(AArch64::MOVZXi);
1017       MovZ.addOperand(MCOperand::createReg(DestReg));
1018       MovZ.addOperand(Hi_MCSym);
1019       MovZ.addOperand(MCOperand::createImm(16));
1020       EmitToStreamer(*OutStreamer, MovZ);
1021
1022       MCInst MovK;
1023       MovK.setOpcode(AArch64::MOVKXi);
1024       MovK.addOperand(MCOperand::createReg(DestReg));
1025       MovK.addOperand(MCOperand::createReg(DestReg));
1026       MovK.addOperand(Lo_MCSym);
1027       MovK.addOperand(MCOperand::createImm(0));
1028       EmitToStreamer(*OutStreamer, MovK);
1029       return;
1030   }
1031   case AArch64::MOVIv2d_ns:
1032     // If the target has <rdar://problem/16473581>, lower this
1033     // instruction to movi.16b instead.
1034     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1035         MI->getOperand(1).getImm() == 0) {
1036       MCInst TmpInst;
1037       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1038       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1039       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1040       EmitToStreamer(*OutStreamer, TmpInst);
1041       return;
1042     }
1043     break;
1044
1045   case AArch64::DBG_VALUE: {
1046     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1047       SmallString<128> TmpStr;
1048       raw_svector_ostream OS(TmpStr);
1049       PrintDebugValueComment(MI, OS);
1050       OutStreamer->EmitRawText(StringRef(OS.str()));
1051     }
1052     return;
1053
1054   case AArch64::EMITBKEY: {
1055       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1056       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1057           ExceptionHandlingType != ExceptionHandling::ARM)
1058         return;
1059
1060       if (needsCFIMoves() == CFI_M_None)
1061         return;
1062
1063       OutStreamer->EmitCFIBKeyFrame();
1064       return;
1065     }
1066   }
1067
1068   // Tail calls use pseudo instructions so they have the proper code-gen
1069   // attributes (isCall, isReturn, etc.). We lower them to the real
1070   // instruction here.
1071   case AArch64::TCRETURNri:
1072   case AArch64::TCRETURNriBTI:
1073   case AArch64::TCRETURNriALL: {
1074     MCInst TmpInst;
1075     TmpInst.setOpcode(AArch64::BR);
1076     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1077     EmitToStreamer(*OutStreamer, TmpInst);
1078     return;
1079   }
1080   case AArch64::TCRETURNdi: {
1081     MCOperand Dest;
1082     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1083     MCInst TmpInst;
1084     TmpInst.setOpcode(AArch64::B);
1085     TmpInst.addOperand(Dest);
1086     EmitToStreamer(*OutStreamer, TmpInst);
1087     return;
1088   }
1089   case AArch64::TLSDESC_CALLSEQ: {
1090     /// lower this to:
1091     ///    adrp  x0, :tlsdesc:var
1092     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1093     ///    add   x0, x0, #:tlsdesc_lo12:var
1094     ///    .tlsdesccall var
1095     ///    blr   x1
1096     ///    (TPIDR_EL0 offset now in x0)
1097     const MachineOperand &MO_Sym = MI->getOperand(0);
1098     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1099     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1100     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1101     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1102     MCInstLowering.lowerOperand(MO_Sym, Sym);
1103     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1104     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1105
1106     MCInst Adrp;
1107     Adrp.setOpcode(AArch64::ADRP);
1108     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1109     Adrp.addOperand(SymTLSDesc);
1110     EmitToStreamer(*OutStreamer, Adrp);
1111
1112     MCInst Ldr;
1113     Ldr.setOpcode(AArch64::LDRXui);
1114     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1115     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1116     Ldr.addOperand(SymTLSDescLo12);
1117     Ldr.addOperand(MCOperand::createImm(0));
1118     EmitToStreamer(*OutStreamer, Ldr);
1119
1120     MCInst Add;
1121     Add.setOpcode(AArch64::ADDXri);
1122     Add.addOperand(MCOperand::createReg(AArch64::X0));
1123     Add.addOperand(MCOperand::createReg(AArch64::X0));
1124     Add.addOperand(SymTLSDescLo12);
1125     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1126     EmitToStreamer(*OutStreamer, Add);
1127
1128     // Emit a relocation-annotation. This expands to no code, but requests
1129     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1130     MCInst TLSDescCall;
1131     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1132     TLSDescCall.addOperand(Sym);
1133     EmitToStreamer(*OutStreamer, TLSDescCall);
1134
1135     MCInst Blr;
1136     Blr.setOpcode(AArch64::BLR);
1137     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1138     EmitToStreamer(*OutStreamer, Blr);
1139
1140     return;
1141   }
1142
1143   case AArch64::JumpTableDest32: {
1144     // We want:
1145     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
1146     //     add xDest, xTable, xScratch
1147     unsigned DestReg = MI->getOperand(0).getReg(),
1148              ScratchReg = MI->getOperand(1).getReg(),
1149              TableReg = MI->getOperand(2).getReg(),
1150              EntryReg = MI->getOperand(3).getReg();
1151     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1152                                      .addReg(ScratchReg)
1153                                      .addReg(TableReg)
1154                                      .addReg(EntryReg)
1155                                      .addImm(0)
1156                                      .addImm(1));
1157     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1158                                      .addReg(DestReg)
1159                                      .addReg(TableReg)
1160                                      .addReg(ScratchReg)
1161                                      .addImm(0));
1162     return;
1163   }
1164   case AArch64::JumpTableDest16:
1165   case AArch64::JumpTableDest8:
1166     LowerJumpTableDestSmall(*OutStreamer, *MI);
1167     return;
1168
1169   case AArch64::FMOVH0:
1170   case AArch64::FMOVS0:
1171   case AArch64::FMOVD0:
1172     EmitFMov0(*MI);
1173     return;
1174
1175   case TargetOpcode::STACKMAP:
1176     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1177
1178   case TargetOpcode::PATCHPOINT:
1179     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1180
1181   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1182     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1183     return;
1184
1185   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1186     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1187     return;
1188
1189   case TargetOpcode::PATCHABLE_TAIL_CALL:
1190     LowerPATCHABLE_TAIL_CALL(*MI);
1191     return;
1192
1193   case AArch64::HWASAN_CHECK_MEMACCESS:
1194   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1195     LowerHWASAN_CHECK_MEMACCESS(*MI);
1196     return;
1197
1198   case AArch64::SEH_StackAlloc:
1199     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1200     return;
1201
1202   case AArch64::SEH_SaveFPLR:
1203     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1204     return;
1205
1206   case AArch64::SEH_SaveFPLR_X:
1207     assert(MI->getOperand(0).getImm() < 0 &&
1208            "Pre increment SEH opcode must have a negative offset");
1209     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1210     return;
1211
1212   case AArch64::SEH_SaveReg:
1213     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1214                                MI->getOperand(1).getImm());
1215     return;
1216
1217   case AArch64::SEH_SaveReg_X:
1218     assert(MI->getOperand(1).getImm() < 0 &&
1219            "Pre increment SEH opcode must have a negative offset");
1220     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1221                                 -MI->getOperand(1).getImm());
1222     return;
1223
1224   case AArch64::SEH_SaveRegP:
1225     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1226             "Non-consecutive registers not allowed for save_regp");
1227     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1228                                 MI->getOperand(2).getImm());
1229     return;
1230
1231   case AArch64::SEH_SaveRegP_X:
1232     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1233             "Non-consecutive registers not allowed for save_regp_x");
1234     assert(MI->getOperand(2).getImm() < 0 &&
1235            "Pre increment SEH opcode must have a negative offset");
1236     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1237                                  -MI->getOperand(2).getImm());
1238     return;
1239
1240   case AArch64::SEH_SaveFReg:
1241     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1242                                 MI->getOperand(1).getImm());
1243     return;
1244
1245   case AArch64::SEH_SaveFReg_X:
1246     assert(MI->getOperand(1).getImm() < 0 &&
1247            "Pre increment SEH opcode must have a negative offset");
1248     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1249                                  -MI->getOperand(1).getImm());
1250     return;
1251
1252   case AArch64::SEH_SaveFRegP:
1253     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1254             "Non-consecutive registers not allowed for save_regp");
1255     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1256                                  MI->getOperand(2).getImm());
1257     return;
1258
1259   case AArch64::SEH_SaveFRegP_X:
1260     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1261             "Non-consecutive registers not allowed for save_regp_x");
1262     assert(MI->getOperand(2).getImm() < 0 &&
1263            "Pre increment SEH opcode must have a negative offset");
1264     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1265                                   -MI->getOperand(2).getImm());
1266     return;
1267
1268   case AArch64::SEH_SetFP:
1269     TS->EmitARM64WinCFISetFP();
1270     return;
1271
1272   case AArch64::SEH_AddFP:
1273     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1274     return;
1275
1276   case AArch64::SEH_Nop:
1277     TS->EmitARM64WinCFINop();
1278     return;
1279
1280   case AArch64::SEH_PrologEnd:
1281     TS->EmitARM64WinCFIPrologEnd();
1282     return;
1283
1284   case AArch64::SEH_EpilogStart:
1285     TS->EmitARM64WinCFIEpilogStart();
1286     return;
1287
1288   case AArch64::SEH_EpilogEnd:
1289     TS->EmitARM64WinCFIEpilogEnd();
1290     return;
1291   }
1292
1293   // Finally, do the automated lowerings for everything else.
1294   MCInst TmpInst;
1295   MCInstLowering.Lower(MI, TmpInst);
1296   EmitToStreamer(*OutStreamer, TmpInst);
1297 }
1298
1299 // Force static initialization.
1300 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1301   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1302   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1303   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1304   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1305   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1306 }