contrib/llvm-project/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

   1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a printer that converts from our internal representation
  10 // of machine-dependent LLVM code to the AArch64 assembly language.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64.h"
  15 #include "AArch64MCInstLower.h"
  16 #include "AArch64MachineFunctionInfo.h"
  17 #include "AArch64RegisterInfo.h"
  18 #include "AArch64Subtarget.h"
  19 #include "AArch64TargetObjectFile.h"
  20 #include "MCTargetDesc/AArch64AddressingModes.h"
  21 #include "MCTargetDesc/AArch64InstPrinter.h"
  22 #include "MCTargetDesc/AArch64MCExpr.h"
  23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
  24 #include "MCTargetDesc/AArch64TargetStreamer.h"
  25 #include "TargetInfo/AArch64TargetInfo.h"
  26 #include "Utils/AArch64BaseInfo.h"
  27 #include "llvm/ADT/SmallString.h"
  28 #include "llvm/ADT/SmallVector.h"
  29 #include "llvm/ADT/StringRef.h"
  30 #include "llvm/ADT/Triple.h"
  31 #include "llvm/ADT/Twine.h"
  32 #include "llvm/BinaryFormat/COFF.h"
  33 #include "llvm/BinaryFormat/ELF.h"
  34 #include "llvm/CodeGen/AsmPrinter.h"
  35 #include "llvm/CodeGen/MachineBasicBlock.h"
  36 #include "llvm/CodeGen/MachineFunction.h"
  37 #include "llvm/CodeGen/MachineInstr.h"
  38 #include "llvm/CodeGen/MachineJumpTableInfo.h"
  39 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
  40 #include "llvm/CodeGen/MachineOperand.h"
  41 #include "llvm/CodeGen/StackMaps.h"
  42 #include "llvm/CodeGen/TargetRegisterInfo.h"
  43 #include "llvm/IR/DataLayout.h"
  44 #include "llvm/IR/DebugInfoMetadata.h"
  45 #include "llvm/MC/MCAsmInfo.h"
  46 #include "llvm/MC/MCContext.h"
  47 #include "llvm/MC/MCInst.h"
  48 #include "llvm/MC/MCInstBuilder.h"
  49 #include "llvm/MC/MCSectionELF.h"
  50 #include "llvm/MC/MCStreamer.h"
  51 #include "llvm/MC/MCSymbol.h"
  52 #include "llvm/Support/Casting.h"
  53 #include "llvm/Support/ErrorHandling.h"
  54 #include "llvm/Support/TargetRegistry.h"
  55 #include "llvm/Support/raw_ostream.h"
  56 #include "llvm/Target/TargetMachine.h"
  57 #include <algorithm>
  58 #include <cassert>
  59 #include <cstdint>
  60 #include <map>
  61 #include <memory>
  62
  63 using namespace llvm;
  64
  65 #define DEBUG_TYPE "asm-printer"
  66
  67 namespace {
  68
  69 class AArch64AsmPrinter : public AsmPrinter {
  70   AArch64MCInstLower MCInstLowering;
  71   StackMaps SM;
  72   const AArch64Subtarget *STI;
  73
  74 public:
  75   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
  76       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
  77         SM(*this) {}
  78
  79   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
  80
  81   /// Wrapper for MCInstLowering.lowerOperand() for the
  82   /// tblgen'erated pseudo lowering.
  83   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
  84     return MCInstLowering.lowerOperand(MO, MCOp);
  85   }
  86
  87   void emitStartOfAsmFile(Module &M) override;
  88   void emitJumpTableInfo() override;
  89   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
  90                           const MachineBasicBlock *MBB, unsigned JTI);
  91
  92   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
  93
  94   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
  95                      const MachineInstr &MI);
  96   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  97                        const MachineInstr &MI);
  98
  99   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
 100   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
 101   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
 102
 103   typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
 104   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
 105   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
 106   void EmitHwasanMemaccessSymbols(Module &M);
 107
 108   void EmitSled(const MachineInstr &MI, SledKind Kind);
 109
 110   /// tblgen'erated driver function for lowering simple MI->MC
 111   /// pseudo instructions.
 112   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
 113                                    const MachineInstr *MI);
 114
 115   void emitInstruction(const MachineInstr *MI) override;
 116
 117   void emitFunctionHeaderComment() override;
 118
 119   void getAnalysisUsage(AnalysisUsage &AU) const override {
 120     AsmPrinter::getAnalysisUsage(AU);
 121     AU.setPreservesAll();
 122   }
 123
 124   bool runOnMachineFunction(MachineFunction &MF) override {
 125     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
 126     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
 127
 128     SetupMachineFunction(MF);
 129
 130     if (STI->isTargetCOFF()) {
 131       bool Internal = MF.getFunction().hasInternalLinkage();
 132       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
 133                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
 134       int Type =
 135         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
 136
 137       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
 138       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
 139       OutStreamer->EmitCOFFSymbolType(Type);
 140       OutStreamer->EndCOFFSymbolDef();
 141     }
 142
 143     // Emit the rest of the function body.
 144     emitFunctionBody();
 145
 146     // Emit the XRay table for this function.
 147     emitXRayTable();
 148
 149     // We didn't modify anything.
 150     return false;
 151   }
 152
 153 private:
 154   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
 155   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
 156   bool printAsmRegInClass(const MachineOperand &MO,
 157                           const TargetRegisterClass *RC, unsigned AltName,
 158                           raw_ostream &O);
 159
 160   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 161                        const char *ExtraCode, raw_ostream &O) override;
 162   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
 163                              const char *ExtraCode, raw_ostream &O) override;
 164
 165   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
 166
 167   void emitFunctionBodyEnd() override;
 168
 169   MCSymbol *GetCPISymbol(unsigned CPID) const override;
 170   void emitEndOfAsmFile(Module &M) override;
 171
 172   AArch64FunctionInfo *AArch64FI = nullptr;
 173
 174   /// Emit the LOHs contained in AArch64FI.
 175   void EmitLOHs();
 176
 177   /// Emit instruction to set float register to zero.
 178   void EmitFMov0(const MachineInstr &MI);
 179
 180   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
 181
 182   MInstToMCSymbol LOHInstToLabel;
 183 };
 184
 185 } // end anonymous namespace
 186
 187 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
 188   if (!TM.getTargetTriple().isOSBinFormatELF())
 189     return;
 190
 191   // Assemble feature flags that may require creation of a note section.
 192   unsigned Flags = ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
 193                    ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
 194
 195   if (any_of(M, [](const Function &F) {
 196         return !F.isDeclaration() &&
 197                !F.hasFnAttribute("branch-target-enforcement");
 198       })) {
 199     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
 200   }
 201
 202   if ((Flags & ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI) == 0 &&
 203       any_of(M, [](const Function &F) {
 204         return F.hasFnAttribute("branch-target-enforcement");
 205       })) {
 206     errs() << "warning: some functions compiled with BTI and some compiled "
 207               "without BTI\n"
 208            << "warning: not setting BTI in feature flags\n";
 209   }
 210
 211   if (any_of(M, [](const Function &F) {
 212         if (F.isDeclaration())
 213           return false;
 214         Attribute A = F.getFnAttribute("sign-return-address");
 215         return !A.isStringAttribute() || A.getValueAsString() == "none";
 216       })) {
 217     Flags &= ~ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
 218   }
 219
 220   if (Flags == 0)
 221     return;
 222
 223   // Emit a .note.gnu.property section with the flags.
 224   MCSection *Cur = OutStreamer->getCurrentSectionOnly();
 225   MCSection *Nt = MMI->getContext().getELFSection(
 226       ".note.gnu.property", ELF::SHT_NOTE, ELF::SHF_ALLOC);
 227   OutStreamer->SwitchSection(Nt);
 228
 229   // Emit the note header.
 230   emitAlignment(Align(8));
 231   OutStreamer->emitInt32(4);     // data size for "GNU\0"
 232   OutStreamer->emitInt32(4 * 4); // Elf_Prop size
 233   OutStreamer->emitInt32(ELF::NT_GNU_PROPERTY_TYPE_0);
 234   OutStreamer->emitBytes(StringRef("GNU", 4)); // note name
 235
 236   // Emit the PAC/BTI properties.
 237   OutStreamer->emitInt32(ELF::GNU_PROPERTY_AARCH64_FEATURE_1_AND);
 238   OutStreamer->emitInt32(4);     // data size
 239   OutStreamer->emitInt32(Flags); // data
 240   OutStreamer->emitInt32(0);     // pad
 241
 242   OutStreamer->endSection(Nt);
 243   OutStreamer->SwitchSection(Cur);
 244 }
 245
 246 void AArch64AsmPrinter::emitFunctionHeaderComment() {
 247   const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
 248   Optional<std::string> OutlinerString = FI->getOutliningStyle();
 249   if (OutlinerString != None)
 250     OutStreamer->GetCommentOS() << ' ' << OutlinerString;
 251 }
 252
 253 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
 254 {
 255   const Function &F = MF->getFunction();
 256   if (F.hasFnAttribute("patchable-function-entry")) {
 257     unsigned Num;
 258     if (F.getFnAttribute("patchable-function-entry")
 259             .getValueAsString()
 260             .getAsInteger(10, Num))
 261       return;
 262     emitNops(Num);
 263     return;
 264   }
 265
 266   EmitSled(MI, SledKind::FUNCTION_ENTER);
 267 }
 268
 269 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
 270 {
 271   EmitSled(MI, SledKind::FUNCTION_EXIT);
 272 }
 273
 274 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
 275 {
 276   EmitSled(MI, SledKind::TAIL_CALL);
 277 }
 278
 279 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
 280 {
 281   static const int8_t NoopsInSledCount = 7;
 282   // We want to emit the following pattern:
 283   //
 284   // .Lxray_sled_N:
 285   //   ALIGN
 286   //   B #32
 287   //   ; 7 NOP instructions (28 bytes)
 288   // .tmpN
 289   //
 290   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
 291   // over the full 32 bytes (8 instructions) with the following pattern:
 292   //
 293   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
 294   //   LDR W0, #12 ; W0 := function ID
 295   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
 296   //   BLR X16 ; call the tracing trampoline
 297   //   ;DATA: 32 bits of function ID
 298   //   ;DATA: lower 32 bits of the address of the trampoline
 299   //   ;DATA: higher 32 bits of the address of the trampoline
 300   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
 301   //
 302   OutStreamer->emitCodeAlignment(4);
 303   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
 304   OutStreamer->emitLabel(CurSled);
 305   auto Target = OutContext.createTempSymbol();
 306
 307   // Emit "B #32" instruction, which jumps over the next 28 bytes.
 308   // The operand has to be the number of 4-byte instructions to jump over,
 309   // including the current instruction.
 310   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
 311
 312   for (int8_t I = 0; I < NoopsInSledCount; I++)
 313     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 314
 315   OutStreamer->emitLabel(Target);
 316   recordSled(CurSled, MI, Kind, 2);
 317 }
 318
 319 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
 320   Register Reg = MI.getOperand(0).getReg();
 321   bool IsShort =
 322       MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
 323   uint32_t AccessInfo = MI.getOperand(1).getImm();
 324   MCSymbol *&Sym =
 325       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
 326   if (!Sym) {
 327     // FIXME: Make this work on non-ELF.
 328     if (!TM.getTargetTriple().isOSBinFormatELF())
 329       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
 330
 331     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
 332                           utostr(AccessInfo);
 333     if (IsShort)
 334       SymName += "_short";
 335     Sym = OutContext.getOrCreateSymbol(SymName);
 336   }
 337
 338   EmitToStreamer(*OutStreamer,
 339                  MCInstBuilder(AArch64::BL)
 340                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
 341 }
 342
 343 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
 344   if (HwasanMemaccessSymbols.empty())
 345     return;
 346
 347   const Triple &TT = TM.getTargetTriple();
 348   assert(TT.isOSBinFormatELF());
 349   std::unique_ptr<MCSubtargetInfo> STI(
 350       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
 351
 352   MCSymbol *HwasanTagMismatchV1Sym =
 353       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
 354   MCSymbol *HwasanTagMismatchV2Sym =
 355       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
 356
 357   const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
 358       MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
 359   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
 360       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
 361
 362   for (auto &P : HwasanMemaccessSymbols) {
 363     unsigned Reg = std::get<0>(P.first);
 364     bool IsShort = std::get<1>(P.first);
 365     uint32_t AccessInfo = std::get<2>(P.first);
 366     const MCSymbolRefExpr *HwasanTagMismatchRef =
 367         IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
 368     MCSymbol *Sym = P.second;
 369
 370     OutStreamer->SwitchSection(OutContext.getELFSection(
 371         ".text.hot", ELF::SHT_PROGBITS,
 372         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
 373         Sym->getName()));
 374
 375     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
 376     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
 377     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
 378     OutStreamer->emitLabel(Sym);
 379
 380     OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
 381                                      .addReg(AArch64::X16)
 382                                      .addReg(Reg)
 383                                      .addImm(4)
 384                                      .addImm(55),
 385                                  *STI);
 386     OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
 387                                      .addReg(AArch64::W16)
 388                                      .addReg(AArch64::X9)
 389                                      .addReg(AArch64::X16)
 390                                      .addImm(0)
 391                                      .addImm(0),
 392                                  *STI);
 393     OutStreamer->emitInstruction(
 394         MCInstBuilder(AArch64::SUBSXrs)
 395             .addReg(AArch64::XZR)
 396             .addReg(AArch64::X16)
 397             .addReg(Reg)
 398             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
 399         *STI);
 400     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
 401     OutStreamer->emitInstruction(
 402         MCInstBuilder(AArch64::Bcc)
 403             .addImm(AArch64CC::NE)
 404             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
 405                                              OutContext)),
 406         *STI);
 407     MCSymbol *ReturnSym = OutContext.createTempSymbol();
 408     OutStreamer->emitLabel(ReturnSym);
 409     OutStreamer->emitInstruction(
 410         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
 411     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
 412
 413     if (IsShort) {
 414       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
 415                                        .addReg(AArch64::WZR)
 416                                        .addReg(AArch64::W16)
 417                                        .addImm(15)
 418                                        .addImm(0),
 419                                    *STI);
 420       MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
 421       OutStreamer->emitInstruction(
 422           MCInstBuilder(AArch64::Bcc)
 423               .addImm(AArch64CC::HI)
 424               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 425           *STI);
 426
 427       OutStreamer->emitInstruction(
 428           MCInstBuilder(AArch64::ANDXri)
 429               .addReg(AArch64::X17)
 430               .addReg(Reg)
 431               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
 432           *STI);
 433       unsigned Size = 1 << (AccessInfo & 0xf);
 434       if (Size != 1)
 435         OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
 436                                          .addReg(AArch64::X17)
 437                                          .addReg(AArch64::X17)
 438                                          .addImm(Size - 1)
 439                                          .addImm(0),
 440                                      *STI);
 441       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
 442                                        .addReg(AArch64::WZR)
 443                                        .addReg(AArch64::W16)
 444                                        .addReg(AArch64::W17)
 445                                        .addImm(0),
 446                                    *STI);
 447       OutStreamer->emitInstruction(
 448           MCInstBuilder(AArch64::Bcc)
 449               .addImm(AArch64CC::LS)
 450               .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 451           *STI);
 452
 453       OutStreamer->emitInstruction(
 454           MCInstBuilder(AArch64::ORRXri)
 455               .addReg(AArch64::X16)
 456               .addReg(Reg)
 457               .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
 458           *STI);
 459       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
 460                                        .addReg(AArch64::W16)
 461                                        .addReg(AArch64::X16)
 462                                        .addImm(0),
 463                                    *STI);
 464       OutStreamer->emitInstruction(
 465           MCInstBuilder(AArch64::SUBSXrs)
 466               .addReg(AArch64::XZR)
 467               .addReg(AArch64::X16)
 468               .addReg(Reg)
 469               .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
 470           *STI);
 471       OutStreamer->emitInstruction(
 472           MCInstBuilder(AArch64::Bcc)
 473               .addImm(AArch64CC::EQ)
 474               .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
 475           *STI);
 476
 477       OutStreamer->emitLabel(HandleMismatchSym);
 478     }
 479
 480     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
 481                                      .addReg(AArch64::SP)
 482                                      .addReg(AArch64::X0)
 483                                      .addReg(AArch64::X1)
 484                                      .addReg(AArch64::SP)
 485                                      .addImm(-32),
 486                                  *STI);
 487     OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
 488                                      .addReg(AArch64::FP)
 489                                      .addReg(AArch64::LR)
 490                                      .addReg(AArch64::SP)
 491                                      .addImm(29),
 492                                  *STI);
 493
 494     if (Reg != AArch64::X0)
 495       OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
 496                                        .addReg(AArch64::X0)
 497                                        .addReg(AArch64::XZR)
 498                                        .addReg(Reg)
 499                                        .addImm(0),
 500                                    *STI);
 501     OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
 502                                      .addReg(AArch64::X1)
 503                                      .addImm(AccessInfo)
 504                                      .addImm(0),
 505                                  *STI);
 506
 507     // Intentionally load the GOT entry and branch to it, rather than possibly
 508     // late binding the function, which may clobber the registers before we have
 509     // a chance to save them.
 510     OutStreamer->emitInstruction(
 511         MCInstBuilder(AArch64::ADRP)
 512             .addReg(AArch64::X16)
 513             .addExpr(AArch64MCExpr::create(
 514                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
 515                 OutContext)),
 516         *STI);
 517     OutStreamer->emitInstruction(
 518         MCInstBuilder(AArch64::LDRXui)
 519             .addReg(AArch64::X16)
 520             .addReg(AArch64::X16)
 521             .addExpr(AArch64MCExpr::create(
 522                 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
 523                 OutContext)),
 524         *STI);
 525     OutStreamer->emitInstruction(
 526         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
 527   }
 528 }
 529
 530 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
 531   EmitHwasanMemaccessSymbols(M);
 532
 533   const Triple &TT = TM.getTargetTriple();
 534   if (TT.isOSBinFormatMachO()) {
 535     // Funny Darwin hack: This flag tells the linker that no global symbols
 536     // contain code that falls through to other global symbols (e.g. the obvious
 537     // implementation of multiple entry points).  If this doesn't occur, the
 538     // linker can safely perform dead code stripping.  Since LLVM never
 539     // generates code that does this, it is always safe to set.
 540     OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
 541   }
 542   emitStackMaps(SM);
 543 }
 544
 545 void AArch64AsmPrinter::EmitLOHs() {
 546   SmallVector<MCSymbol *, 3> MCArgs;
 547
 548   for (const auto &D : AArch64FI->getLOHContainer()) {
 549     for (const MachineInstr *MI : D.getArgs()) {
 550       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
 551       assert(LabelIt != LOHInstToLabel.end() &&
 552              "Label hasn't been inserted for LOH related instruction");
 553       MCArgs.push_back(LabelIt->second);
 554     }
 555     OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
 556     MCArgs.clear();
 557   }
 558 }
 559
 560 void AArch64AsmPrinter::emitFunctionBodyEnd() {
 561   if (!AArch64FI->getLOHRelated().empty())
 562     EmitLOHs();
 563 }
 564
 565 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
 566 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
 567   // Darwin uses a linker-private symbol name for constant-pools (to
 568   // avoid addends on the relocation?), ELF has no such concept and
 569   // uses a normal private symbol.
 570   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
 571     return OutContext.getOrCreateSymbol(
 572         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
 573         Twine(getFunctionNumber()) + "_" + Twine(CPID));
 574
 575   return AsmPrinter::GetCPISymbol(CPID);
 576 }
 577
 578 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
 579                                      raw_ostream &O) {
 580   const MachineOperand &MO = MI->getOperand(OpNum);
 581   switch (MO.getType()) {
 582   default:
 583     llvm_unreachable("<unknown operand type>");
 584   case MachineOperand::MO_Register: {
 585     Register Reg = MO.getReg();
 586     assert(Register::isPhysicalRegister(Reg));
 587     assert(!MO.getSubReg() && "Subregs should be eliminated!");
 588     O << AArch64InstPrinter::getRegisterName(Reg);
 589     break;
 590   }
 591   case MachineOperand::MO_Immediate: {
 592     O << MO.getImm();
 593     break;
 594   }
 595   case MachineOperand::MO_GlobalAddress: {
 596     PrintSymbolOperand(MO, O);
 597     break;
 598   }
 599   case MachineOperand::MO_BlockAddress: {
 600     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
 601     Sym->print(O, MAI);
 602     break;
 603   }
 604   }
 605 }
 606
 607 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
 608                                           raw_ostream &O) {
 609   Register Reg = MO.getReg();
 610   switch (Mode) {
 611   default:
 612     return true; // Unknown mode.
 613   case 'w':
 614     Reg = getWRegFromXReg(Reg);
 615     break;
 616   case 'x':
 617     Reg = getXRegFromWReg(Reg);
 618     break;
 619   }
 620
 621   O << AArch64InstPrinter::getRegisterName(Reg);
 622   return false;
 623 }
 624
 625 // Prints the register in MO using class RC using the offset in the
 626 // new register class. This should not be used for cross class
 627 // printing.
 628 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
 629                                            const TargetRegisterClass *RC,
 630                                            unsigned AltName, raw_ostream &O) {
 631   assert(MO.isReg() && "Should only get here with a register!");
 632   const TargetRegisterInfo *RI = STI->getRegisterInfo();
 633   Register Reg = MO.getReg();
 634   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
 635   assert(RI->regsOverlap(RegToPrint, Reg));
 636   O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
 637   return false;
 638 }
 639
 640 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 641                                         const char *ExtraCode, raw_ostream &O) {
 642   const MachineOperand &MO = MI->getOperand(OpNum);
 643
 644   // First try the generic code, which knows about modifiers like 'c' and 'n'.
 645   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
 646     return false;
 647
 648   // Does this asm operand have a single letter operand modifier?
 649   if (ExtraCode && ExtraCode[0]) {
 650     if (ExtraCode[1] != 0)
 651       return true; // Unknown modifier.
 652
 653     switch (ExtraCode[0]) {
 654     default:
 655       return true; // Unknown modifier.
 656     case 'w':      // Print W register
 657     case 'x':      // Print X register
 658       if (MO.isReg())
 659         return printAsmMRegister(MO, ExtraCode[0], O);
 660       if (MO.isImm() && MO.getImm() == 0) {
 661         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
 662         O << AArch64InstPrinter::getRegisterName(Reg);
 663         return false;
 664       }
 665       printOperand(MI, OpNum, O);
 666       return false;
 667     case 'b': // Print B register.
 668     case 'h': // Print H register.
 669     case 's': // Print S register.
 670     case 'd': // Print D register.
 671     case 'q': // Print Q register.
 672     case 'z': // Print Z register.
 673       if (MO.isReg()) {
 674         const TargetRegisterClass *RC;
 675         switch (ExtraCode[0]) {
 676         case 'b':
 677           RC = &AArch64::FPR8RegClass;
 678           break;
 679         case 'h':
 680           RC = &AArch64::FPR16RegClass;
 681           break;
 682         case 's':
 683           RC = &AArch64::FPR32RegClass;
 684           break;
 685         case 'd':
 686           RC = &AArch64::FPR64RegClass;
 687           break;
 688         case 'q':
 689           RC = &AArch64::FPR128RegClass;
 690           break;
 691         case 'z':
 692           RC = &AArch64::ZPRRegClass;
 693           break;
 694         default:
 695           return true;
 696         }
 697         return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
 698       }
 699       printOperand(MI, OpNum, O);
 700       return false;
 701     }
 702   }
 703
 704   // According to ARM, we should emit x and v registers unless we have a
 705   // modifier.
 706   if (MO.isReg()) {
 707     Register Reg = MO.getReg();
 708
 709     // If this is a w or x register, print an x register.
 710     if (AArch64::GPR32allRegClass.contains(Reg) ||
 711         AArch64::GPR64allRegClass.contains(Reg))
 712       return printAsmMRegister(MO, 'x', O);
 713
 714     unsigned AltName = AArch64::NoRegAltName;
 715     const TargetRegisterClass *RegClass;
 716     if (AArch64::ZPRRegClass.contains(Reg)) {
 717       RegClass = &AArch64::ZPRRegClass;
 718     } else if (AArch64::PPRRegClass.contains(Reg)) {
 719       RegClass = &AArch64::PPRRegClass;
 720     } else {
 721       RegClass = &AArch64::FPR128RegClass;
 722       AltName = AArch64::vreg;
 723     }
 724
 725     // If this is a b, h, s, d, or q register, print it as a v register.
 726     return printAsmRegInClass(MO, RegClass, AltName, O);
 727   }
 728
 729   printOperand(MI, OpNum, O);
 730   return false;
 731 }
 732
 733 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 734                                               unsigned OpNum,
 735                                               const char *ExtraCode,
 736                                               raw_ostream &O) {
 737   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
 738     return true; // Unknown modifier.
 739
 740   const MachineOperand &MO = MI->getOperand(OpNum);
 741   assert(MO.isReg() && "unexpected inline asm memory operand");
 742   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
 743   return false;
 744 }
 745
 746 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
 747                                                raw_ostream &OS) {
 748   unsigned NOps = MI->getNumOperands();
 749   assert(NOps == 4);
 750   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
 751   // cast away const; DIetc do not take const operands for some reason.
 752   OS << MI->getDebugVariable()->getName();
 753   OS << " <- ";
 754   // Frame address.  Currently handles register +- offset only.
 755   assert(MI->getDebugOperand(0).isReg() && MI->isDebugOffsetImm());
 756   OS << '[';
 757   printOperand(MI, 0, OS);
 758   OS << '+';
 759   printOperand(MI, 1, OS);
 760   OS << ']';
 761   OS << "+";
 762   printOperand(MI, NOps - 2, OS);
 763 }
 764
 765 void AArch64AsmPrinter::emitJumpTableInfo() {
 766   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
 767   if (!MJTI) return;
 768
 769   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
 770   if (JT.empty()) return;
 771
 772   const Function &F = MF->getFunction();
 773   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
 774   bool JTInDiffSection =
 775       !STI->isTargetCOFF() ||
 776       !TLOF.shouldPutJumpTableInFunctionSection(
 777           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
 778           F);
 779   if (JTInDiffSection) {
 780       // Drop it in the readonly section.
 781       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
 782       OutStreamer->SwitchSection(ReadOnlySec);
 783   }
 784
 785   auto AFI = MF->getInfo<AArch64FunctionInfo>();
 786   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
 787     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
 788
 789     // If this jump table was deleted, ignore it.
 790     if (JTBBs.empty()) continue;
 791
 792     unsigned Size = AFI->getJumpTableEntrySize(JTI);
 793     emitAlignment(Align(Size));
 794     OutStreamer->emitLabel(GetJTISymbol(JTI));
 795
 796     for (auto *JTBB : JTBBs)
 797       emitJumpTableEntry(MJTI, JTBB, JTI);
 798   }
 799 }
 800
 801 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
 802                                            const MachineBasicBlock *MBB,
 803                                            unsigned JTI) {
 804   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
 805   auto AFI = MF->getInfo<AArch64FunctionInfo>();
 806   unsigned Size = AFI->getJumpTableEntrySize(JTI);
 807
 808   if (Size == 4) {
 809     // .word LBB - LJTI
 810     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
 811     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
 812     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
 813   } else {
 814     // .byte (LBB - LBB) >> 2 (or .hword)
 815     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
 816     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
 817     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
 818     Value = MCBinaryExpr::createLShr(
 819         Value, MCConstantExpr::create(2, OutContext), OutContext);
 820   }
 821
 822   OutStreamer->emitValue(Value, Size);
 823 }
 824
 825 /// Small jump tables contain an unsigned byte or half, representing the offset
 826 /// from the lowest-addressed possible destination to the desired basic
 827 /// block. Since all instructions are 4-byte aligned, this is further compressed
 828 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
 829 /// materialize the correct destination we need:
 830 ///
 831 ///             adr xDest, .LBB0_0
 832 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
 833 ///             add xDest, xDest, xScratch, lsl #2
 834 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
 835                                                 const llvm::MachineInstr &MI) {
 836   Register DestReg = MI.getOperand(0).getReg();
 837   Register ScratchReg = MI.getOperand(1).getReg();
 838   Register ScratchRegW =
 839       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
 840   Register TableReg = MI.getOperand(2).getReg();
 841   Register EntryReg = MI.getOperand(3).getReg();
 842   int JTIdx = MI.getOperand(4).getIndex();
 843   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
 844
 845   // This has to be first because the compression pass based its reachability
 846   // calculations on the start of the JumpTableDest instruction.
 847   auto Label =
 848       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
 849   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
 850                                   .addReg(DestReg)
 851                                   .addExpr(MCSymbolRefExpr::create(
 852                                       Label, MF->getContext())));
 853
 854   // Load the number of instruction-steps to offset from the label.
 855   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
 856   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
 857                                   .addReg(ScratchRegW)
 858                                   .addReg(TableReg)
 859                                   .addReg(EntryReg)
 860                                   .addImm(0)
 861                                   .addImm(IsByteEntry ? 0 : 1));
 862
 863   // Multiply the steps by 4 and add to the already materialized base label
 864   // address.
 865   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
 866                                   .addReg(DestReg)
 867                                   .addReg(DestReg)
 868                                   .addReg(ScratchReg)
 869                                   .addImm(2));
 870 }
 871
 872 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
 873                                       const MachineInstr &MI) {
 874   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
 875
 876   auto &Ctx = OutStreamer.getContext();
 877   MCSymbol *MILabel = Ctx.createTempSymbol();
 878   OutStreamer.emitLabel(MILabel);
 879
 880   SM.recordStackMap(*MILabel, MI);
 881   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
 882
 883   // Scan ahead to trim the shadow.
 884   const MachineBasicBlock &MBB = *MI.getParent();
 885   MachineBasicBlock::const_iterator MII(MI);
 886   ++MII;
 887   while (NumNOPBytes > 0) {
 888     if (MII == MBB.end() || MII->isCall() ||
 889         MII->getOpcode() == AArch64::DBG_VALUE ||
 890         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
 891         MII->getOpcode() == TargetOpcode::STACKMAP)
 892       break;
 893     ++MII;
 894     NumNOPBytes -= 4;
 895   }
 896
 897   // Emit nops.
 898   for (unsigned i = 0; i < NumNOPBytes; i += 4)
 899     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 900 }
 901
 902 // Lower a patchpoint of the form:
 903 // [<def>], <id>, <numBytes>, <target>, <numArgs>
 904 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 905                                         const MachineInstr &MI) {
 906   auto &Ctx = OutStreamer.getContext();
 907   MCSymbol *MILabel = Ctx.createTempSymbol();
 908   OutStreamer.emitLabel(MILabel);
 909   SM.recordPatchPoint(*MILabel, MI);
 910
 911   PatchPointOpers Opers(&MI);
 912
 913   int64_t CallTarget = Opers.getCallTarget().getImm();
 914   unsigned EncodedBytes = 0;
 915   if (CallTarget) {
 916     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
 917            "High 16 bits of call target should be zero.");
 918     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
 919     EncodedBytes = 16;
 920     // Materialize the jump address:
 921     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
 922                                     .addReg(ScratchReg)
 923                                     .addImm((CallTarget >> 32) & 0xFFFF)
 924                                     .addImm(32));
 925     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
 926                                     .addReg(ScratchReg)
 927                                     .addReg(ScratchReg)
 928                                     .addImm((CallTarget >> 16) & 0xFFFF)
 929                                     .addImm(16));
 930     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
 931                                     .addReg(ScratchReg)
 932                                     .addReg(ScratchReg)
 933                                     .addImm(CallTarget & 0xFFFF)
 934                                     .addImm(0));
 935     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
 936   }
 937   // Emit padding.
 938   unsigned NumBytes = Opers.getNumPatchBytes();
 939   assert(NumBytes >= EncodedBytes &&
 940          "Patchpoint can't request size less than the length of a call.");
 941   assert((NumBytes - EncodedBytes) % 4 == 0 &&
 942          "Invalid number of NOP bytes requested!");
 943   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
 944     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 945 }
 946
 947 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
 948   Register DestReg = MI.getOperand(0).getReg();
 949   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
 950     // Convert H/S/D register to corresponding Q register
 951     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
 952       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
 953     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
 954       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
 955     else {
 956       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
 957       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
 958     }
 959     MCInst MOVI;
 960     MOVI.setOpcode(AArch64::MOVIv2d_ns);
 961     MOVI.addOperand(MCOperand::createReg(DestReg));
 962     MOVI.addOperand(MCOperand::createImm(0));
 963     EmitToStreamer(*OutStreamer, MOVI);
 964   } else {
 965     MCInst FMov;
 966     switch (MI.getOpcode()) {
 967     default: llvm_unreachable("Unexpected opcode");
 968     case AArch64::FMOVH0:
 969       FMov.setOpcode(AArch64::FMOVWHr);
 970       FMov.addOperand(MCOperand::createReg(DestReg));
 971       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
 972       break;
 973     case AArch64::FMOVS0:
 974       FMov.setOpcode(AArch64::FMOVWSr);
 975       FMov.addOperand(MCOperand::createReg(DestReg));
 976       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
 977       break;
 978     case AArch64::FMOVD0:
 979       FMov.setOpcode(AArch64::FMOVXDr);
 980       FMov.addOperand(MCOperand::createReg(DestReg));
 981       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
 982       break;
 983     }
 984     EmitToStreamer(*OutStreamer, FMov);
 985   }
 986 }
 987
 988 // Simple pseudo-instructions have their lowering (with expansion to real
 989 // instructions) auto-generated.
 990 #include "AArch64GenMCPseudoLowering.inc"
 991
 992 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
 993   // Do any auto-generated pseudo lowerings.
 994   if (emitPseudoExpansionLowering(*OutStreamer, MI))
 995     return;
 996
 997   if (AArch64FI->getLOHRelated().count(MI)) {
 998     // Generate a label for LOH related instruction
 999     MCSymbol *LOHLabel = createTempSymbol("loh");
1000     // Associate the instruction with the label
1001     LOHInstToLabel[MI] = LOHLabel;
1002     OutStreamer->emitLabel(LOHLabel);
1003   }
1004
1005   AArch64TargetStreamer *TS =
1006     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1007   // Do any manual lowerings.
1008   switch (MI->getOpcode()) {
1009   default:
1010     break;
1011   case AArch64::HINT: {
1012     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1013     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1014     // non-empty. If MI is the initial BTI, place the
1015     // __patchable_function_entries label after BTI.
1016     if (CurrentPatchableFunctionEntrySym &&
1017         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1018         MI == &MF->front().front()) {
1019       int64_t Imm = MI->getOperand(0).getImm();
1020       if ((Imm & 32) && (Imm & 6)) {
1021         MCInst Inst;
1022         MCInstLowering.Lower(MI, Inst);
1023         EmitToStreamer(*OutStreamer, Inst);
1024         CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1025         OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1026         return;
1027       }
1028     }
1029     break;
1030   }
1031     case AArch64::MOVMCSym: {
1032       Register DestReg = MI->getOperand(0).getReg();
1033       const MachineOperand &MO_Sym = MI->getOperand(1);
1034       MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1035       MCOperand Hi_MCSym, Lo_MCSym;
1036
1037       Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1038       Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1039
1040       MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1041       MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1042
1043       MCInst MovZ;
1044       MovZ.setOpcode(AArch64::MOVZXi);
1045       MovZ.addOperand(MCOperand::createReg(DestReg));
1046       MovZ.addOperand(Hi_MCSym);
1047       MovZ.addOperand(MCOperand::createImm(16));
1048       EmitToStreamer(*OutStreamer, MovZ);
1049
1050       MCInst MovK;
1051       MovK.setOpcode(AArch64::MOVKXi);
1052       MovK.addOperand(MCOperand::createReg(DestReg));
1053       MovK.addOperand(MCOperand::createReg(DestReg));
1054       MovK.addOperand(Lo_MCSym);
1055       MovK.addOperand(MCOperand::createImm(0));
1056       EmitToStreamer(*OutStreamer, MovK);
1057       return;
1058   }
1059   case AArch64::MOVIv2d_ns:
1060     // If the target has <rdar://problem/16473581>, lower this
1061     // instruction to movi.16b instead.
1062     if (STI->hasZeroCycleZeroingFPWorkaround() &&
1063         MI->getOperand(1).getImm() == 0) {
1064       MCInst TmpInst;
1065       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1066       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1067       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1068       EmitToStreamer(*OutStreamer, TmpInst);
1069       return;
1070     }
1071     break;
1072
1073   case AArch64::DBG_VALUE: {
1074     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1075       SmallString<128> TmpStr;
1076       raw_svector_ostream OS(TmpStr);
1077       PrintDebugValueComment(MI, OS);
1078       OutStreamer->emitRawText(StringRef(OS.str()));
1079     }
1080     return;
1081
1082   case AArch64::EMITBKEY: {
1083       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1084       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1085           ExceptionHandlingType != ExceptionHandling::ARM)
1086         return;
1087
1088       if (needsCFIMoves() == CFI_M_None)
1089         return;
1090
1091       OutStreamer->emitCFIBKeyFrame();
1092       return;
1093     }
1094   }
1095
1096   // Tail calls use pseudo instructions so they have the proper code-gen
1097   // attributes (isCall, isReturn, etc.). We lower them to the real
1098   // instruction here.
1099   case AArch64::TCRETURNri:
1100   case AArch64::TCRETURNriBTI:
1101   case AArch64::TCRETURNriALL: {
1102     MCInst TmpInst;
1103     TmpInst.setOpcode(AArch64::BR);
1104     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1105     EmitToStreamer(*OutStreamer, TmpInst);
1106     return;
1107   }
1108   case AArch64::TCRETURNdi: {
1109     MCOperand Dest;
1110     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1111     MCInst TmpInst;
1112     TmpInst.setOpcode(AArch64::B);
1113     TmpInst.addOperand(Dest);
1114     EmitToStreamer(*OutStreamer, TmpInst);
1115     return;
1116   }
1117   case AArch64::SpeculationBarrierISBDSBEndBB: {
1118     // Print DSB SYS + ISB
1119     MCInst TmpInstDSB;
1120     TmpInstDSB.setOpcode(AArch64::DSB);
1121     TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1122     EmitToStreamer(*OutStreamer, TmpInstDSB);
1123     MCInst TmpInstISB;
1124     TmpInstISB.setOpcode(AArch64::ISB);
1125     TmpInstISB.addOperand(MCOperand::createImm(0xf));
1126     EmitToStreamer(*OutStreamer, TmpInstISB);
1127     return;
1128   }
1129   case AArch64::SpeculationBarrierSBEndBB: {
1130     // Print SB
1131     MCInst TmpInstSB;
1132     TmpInstSB.setOpcode(AArch64::SB);
1133     EmitToStreamer(*OutStreamer, TmpInstSB);
1134     return;
1135   }
1136   case AArch64::TLSDESC_CALLSEQ: {
1137     /// lower this to:
1138     ///    adrp  x0, :tlsdesc:var
1139     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
1140     ///    add   x0, x0, #:tlsdesc_lo12:var
1141     ///    .tlsdesccall var
1142     ///    blr   x1
1143     ///    (TPIDR_EL0 offset now in x0)
1144     const MachineOperand &MO_Sym = MI->getOperand(0);
1145     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1146     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1147     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1148     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1149     MCInstLowering.lowerOperand(MO_Sym, Sym);
1150     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1151     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1152
1153     MCInst Adrp;
1154     Adrp.setOpcode(AArch64::ADRP);
1155     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1156     Adrp.addOperand(SymTLSDesc);
1157     EmitToStreamer(*OutStreamer, Adrp);
1158
1159     MCInst Ldr;
1160     Ldr.setOpcode(AArch64::LDRXui);
1161     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1162     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1163     Ldr.addOperand(SymTLSDescLo12);
1164     Ldr.addOperand(MCOperand::createImm(0));
1165     EmitToStreamer(*OutStreamer, Ldr);
1166
1167     MCInst Add;
1168     Add.setOpcode(AArch64::ADDXri);
1169     Add.addOperand(MCOperand::createReg(AArch64::X0));
1170     Add.addOperand(MCOperand::createReg(AArch64::X0));
1171     Add.addOperand(SymTLSDescLo12);
1172     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1173     EmitToStreamer(*OutStreamer, Add);
1174
1175     // Emit a relocation-annotation. This expands to no code, but requests
1176     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1177     MCInst TLSDescCall;
1178     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1179     TLSDescCall.addOperand(Sym);
1180     EmitToStreamer(*OutStreamer, TLSDescCall);
1181
1182     MCInst Blr;
1183     Blr.setOpcode(AArch64::BLR);
1184     Blr.addOperand(MCOperand::createReg(AArch64::X1));
1185     EmitToStreamer(*OutStreamer, Blr);
1186
1187     return;
1188   }
1189
1190   case AArch64::JumpTableDest32: {
1191     // We want:
1192     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
1193     //     add xDest, xTable, xScratch
1194     unsigned DestReg = MI->getOperand(0).getReg(),
1195              ScratchReg = MI->getOperand(1).getReg(),
1196              TableReg = MI->getOperand(2).getReg(),
1197              EntryReg = MI->getOperand(3).getReg();
1198     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1199                                      .addReg(ScratchReg)
1200                                      .addReg(TableReg)
1201                                      .addReg(EntryReg)
1202                                      .addImm(0)
1203                                      .addImm(1));
1204     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1205                                      .addReg(DestReg)
1206                                      .addReg(TableReg)
1207                                      .addReg(ScratchReg)
1208                                      .addImm(0));
1209     return;
1210   }
1211   case AArch64::JumpTableDest16:
1212   case AArch64::JumpTableDest8:
1213     LowerJumpTableDestSmall(*OutStreamer, *MI);
1214     return;
1215
1216   case AArch64::FMOVH0:
1217   case AArch64::FMOVS0:
1218   case AArch64::FMOVD0:
1219     EmitFMov0(*MI);
1220     return;
1221
1222   case TargetOpcode::STACKMAP:
1223     return LowerSTACKMAP(*OutStreamer, SM, *MI);
1224
1225   case TargetOpcode::PATCHPOINT:
1226     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1227
1228   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1229     LowerPATCHABLE_FUNCTION_ENTER(*MI);
1230     return;
1231
1232   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1233     LowerPATCHABLE_FUNCTION_EXIT(*MI);
1234     return;
1235
1236   case TargetOpcode::PATCHABLE_TAIL_CALL:
1237     LowerPATCHABLE_TAIL_CALL(*MI);
1238     return;
1239
1240   case AArch64::HWASAN_CHECK_MEMACCESS:
1241   case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1242     LowerHWASAN_CHECK_MEMACCESS(*MI);
1243     return;
1244
1245   case AArch64::SEH_StackAlloc:
1246     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1247     return;
1248
1249   case AArch64::SEH_SaveFPLR:
1250     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1251     return;
1252
1253   case AArch64::SEH_SaveFPLR_X:
1254     assert(MI->getOperand(0).getImm() < 0 &&
1255            "Pre increment SEH opcode must have a negative offset");
1256     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1257     return;
1258
1259   case AArch64::SEH_SaveReg:
1260     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1261                                MI->getOperand(1).getImm());
1262     return;
1263
1264   case AArch64::SEH_SaveReg_X:
1265     assert(MI->getOperand(1).getImm() < 0 &&
1266            "Pre increment SEH opcode must have a negative offset");
1267     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1268                                 -MI->getOperand(1).getImm());
1269     return;
1270
1271   case AArch64::SEH_SaveRegP:
1272     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1273             "Non-consecutive registers not allowed for save_regp");
1274     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1275                                 MI->getOperand(2).getImm());
1276     return;
1277
1278   case AArch64::SEH_SaveRegP_X:
1279     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1280             "Non-consecutive registers not allowed for save_regp_x");
1281     assert(MI->getOperand(2).getImm() < 0 &&
1282            "Pre increment SEH opcode must have a negative offset");
1283     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1284                                  -MI->getOperand(2).getImm());
1285     return;
1286
1287   case AArch64::SEH_SaveFReg:
1288     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1289                                 MI->getOperand(1).getImm());
1290     return;
1291
1292   case AArch64::SEH_SaveFReg_X:
1293     assert(MI->getOperand(1).getImm() < 0 &&
1294            "Pre increment SEH opcode must have a negative offset");
1295     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1296                                  -MI->getOperand(1).getImm());
1297     return;
1298
1299   case AArch64::SEH_SaveFRegP:
1300     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1301             "Non-consecutive registers not allowed for save_regp");
1302     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1303                                  MI->getOperand(2).getImm());
1304     return;
1305
1306   case AArch64::SEH_SaveFRegP_X:
1307     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1308             "Non-consecutive registers not allowed for save_regp_x");
1309     assert(MI->getOperand(2).getImm() < 0 &&
1310            "Pre increment SEH opcode must have a negative offset");
1311     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1312                                   -MI->getOperand(2).getImm());
1313     return;
1314
1315   case AArch64::SEH_SetFP:
1316     TS->EmitARM64WinCFISetFP();
1317     return;
1318
1319   case AArch64::SEH_AddFP:
1320     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1321     return;
1322
1323   case AArch64::SEH_Nop:
1324     TS->EmitARM64WinCFINop();
1325     return;
1326
1327   case AArch64::SEH_PrologEnd:
1328     TS->EmitARM64WinCFIPrologEnd();
1329     return;
1330
1331   case AArch64::SEH_EpilogStart:
1332     TS->EmitARM64WinCFIEpilogStart();
1333     return;
1334
1335   case AArch64::SEH_EpilogEnd:
1336     TS->EmitARM64WinCFIEpilogEnd();
1337     return;
1338   }
1339
1340   // Finally, do the automated lowerings for everything else.
1341   MCInst TmpInst;
1342   MCInstLowering.Lower(MI, TmpInst);
1343   EmitToStreamer(*OutStreamer, TmpInst);
1344 }
1345
1346 // Force static initialization.
1347 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1348   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1349   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1350   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1351   RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1352   RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1353 }