contrib/llvm-project/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

   1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file declares the API for the instruction selector.
  10 /// This class is responsible for selecting machine instructions.
  11 /// It's implemented by the target. It's used by the InstructionSelect pass.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  17
  18 #include "llvm/ADT/SmallVector.h"
  19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  21 #include "llvm/CodeGen/GlobalISel/Utils.h"
  22 #include "llvm/CodeGen/MachineInstrBuilder.h"
  23 #include "llvm/CodeGen/MachineOperand.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetInstrInfo.h"
  26 #include "llvm/CodeGen/TargetOpcodes.h"
  27 #include "llvm/CodeGen/TargetRegisterInfo.h"
  28 #include "llvm/IR/Constants.h"
  29 #include "llvm/IR/DataLayout.h"
  30 #include "llvm/Support/Debug.h"
  31 #include "llvm/Support/ErrorHandling.h"
  32 #include "llvm/Support/raw_ostream.h"
  33 #include <cassert>
  34 #include <cstddef>
  35 #include <cstdint>
  36
  37 namespace llvm {
  38
  39 /// GlobalISel PatFrag Predicates
  40 enum {
  41   GIPFP_I64_Invalid = 0,
  42   GIPFP_APInt_Invalid = 0,
  43   GIPFP_APFloat_Invalid = 0,
  44   GIPFP_MI_Invalid = 0,
  45 };
  46
  47 template <class TgtInstructionSelector, class PredicateBitset,
  48           class ComplexMatcherMemFn, class CustomRendererFn>
  49 bool InstructionSelector::executeMatchTable(
  50     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
  51     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
  52         &ISelInfo,
  53     const int64_t *MatchTable, const TargetInstrInfo &TII,
  54     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
  55     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
  56     CodeGenCoverage &CoverageInfo) const {
  57
  58   uint64_t CurrentIdx = 0;
  59   SmallVector<uint64_t, 4> OnFailResumeAt;
  60
  61   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
  62   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
  63
  64   const uint16_t Flags = State.MIs[0]->getFlags();
  65
  66   enum RejectAction { RejectAndGiveUp, RejectAndResume };
  67   auto handleReject = [&]() -> RejectAction {
  68     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  69                     dbgs() << CurrentIdx << ": Rejected\n");
  70     if (OnFailResumeAt.empty())
  71       return RejectAndGiveUp;
  72     CurrentIdx = OnFailResumeAt.pop_back_val();
  73     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  74                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
  75                            << OnFailResumeAt.size() << " try-blocks remain)\n");
  76     return RejectAndResume;
  77   };
  78
  79   auto propagateFlags = [=](NewMIVector &OutMIs) {
  80     for (auto MIB : OutMIs) {
  81       // Set the NoFPExcept flag when no original matched instruction could
  82       // raise an FP exception, but the new instruction potentially might.
  83       uint16_t MIBFlags = Flags;
  84       if (NoFPException && MIB->mayRaiseFPException())
  85         MIBFlags |= MachineInstr::NoFPExcept;
  86       MIB.setMIFlags(MIBFlags);
  87     }
  88
  89     return true;
  90   };
  91
  92   while (true) {
  93     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
  94     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
  95     switch (MatcherOpcode) {
  96     case GIM_Try: {
  97       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  98                       dbgs() << CurrentIdx << ": Begin try-block\n");
  99       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
 100       break;
 101     }
 102
 103     case GIM_RecordInsn: {
 104       int64_t NewInsnID = MatchTable[CurrentIdx++];
 105       int64_t InsnID = MatchTable[CurrentIdx++];
 106       int64_t OpIdx = MatchTable[CurrentIdx++];
 107
 108       // As an optimisation we require that MIs[0] is always the root. Refuse
 109       // any attempt to modify it.
 110       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
 111
 112       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 113       if (!MO.isReg()) {
 114         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 115                         dbgs() << CurrentIdx << ": Not a register\n");
 116         if (handleReject() == RejectAndGiveUp)
 117           return false;
 118         break;
 119       }
 120       if (Register::isPhysicalRegister(MO.getReg())) {
 121         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 122                         dbgs() << CurrentIdx << ": Is a physical register\n");
 123         if (handleReject() == RejectAndGiveUp)
 124           return false;
 125         break;
 126       }
 127
 128       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
 129       if ((size_t)NewInsnID < State.MIs.size())
 130         State.MIs[NewInsnID] = NewMI;
 131       else {
 132         assert((size_t)NewInsnID == State.MIs.size() &&
 133                "Expected to store MIs in order");
 134         State.MIs.push_back(NewMI);
 135       }
 136       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 137                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
 138                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
 139                              << ")\n");
 140       break;
 141     }
 142
 143     case GIM_CheckFeatures: {
 144       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
 145       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 146                       dbgs() << CurrentIdx
 147                              << ": GIM_CheckFeatures(ExpectedBitsetID="
 148                              << ExpectedBitsetID << ")\n");
 149       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
 150           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
 151         if (handleReject() == RejectAndGiveUp)
 152           return false;
 153       }
 154       break;
 155     }
 156
 157     case GIM_CheckOpcode: {
 158       int64_t InsnID = MatchTable[CurrentIdx++];
 159       int64_t Expected = MatchTable[CurrentIdx++];
 160
 161       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 162       unsigned Opcode = State.MIs[InsnID]->getOpcode();
 163
 164       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 165                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
 166                              << "], ExpectedOpcode=" << Expected
 167                              << ") // Got=" << Opcode << "\n");
 168       if (Opcode != Expected) {
 169         if (handleReject() == RejectAndGiveUp)
 170           return false;
 171       }
 172       break;
 173     }
 174
 175     case GIM_SwitchOpcode: {
 176       int64_t InsnID = MatchTable[CurrentIdx++];
 177       int64_t LowerBound = MatchTable[CurrentIdx++];
 178       int64_t UpperBound = MatchTable[CurrentIdx++];
 179       int64_t Default = MatchTable[CurrentIdx++];
 180
 181       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 182       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
 183
 184       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 185         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
 186                << LowerBound << ", " << UpperBound << "), Default=" << Default
 187                << ", JumpTable...) // Got=" << Opcode << "\n";
 188       });
 189       if (Opcode < LowerBound || UpperBound <= Opcode) {
 190         CurrentIdx = Default;
 191         break;
 192       }
 193       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
 194       if (!CurrentIdx) {
 195         CurrentIdx = Default;
 196         break;
 197       }
 198       OnFailResumeAt.push_back(Default);
 199       break;
 200     }
 201
 202     case GIM_SwitchType: {
 203       int64_t InsnID = MatchTable[CurrentIdx++];
 204       int64_t OpIdx = MatchTable[CurrentIdx++];
 205       int64_t LowerBound = MatchTable[CurrentIdx++];
 206       int64_t UpperBound = MatchTable[CurrentIdx++];
 207       int64_t Default = MatchTable[CurrentIdx++];
 208
 209       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 210       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 211
 212       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 213         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
 214                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
 215                << UpperBound << "), Default=" << Default
 216                << ", JumpTable...) // Got=";
 217         if (!MO.isReg())
 218           dbgs() << "Not a VReg\n";
 219         else
 220           dbgs() << MRI.getType(MO.getReg()) << "\n";
 221       });
 222       if (!MO.isReg()) {
 223         CurrentIdx = Default;
 224         break;
 225       }
 226       const LLT Ty = MRI.getType(MO.getReg());
 227       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
 228       if (TyI == ISelInfo.TypeIDMap.end()) {
 229         CurrentIdx = Default;
 230         break;
 231       }
 232       const int64_t TypeID = TyI->second;
 233       if (TypeID < LowerBound || UpperBound <= TypeID) {
 234         CurrentIdx = Default;
 235         break;
 236       }
 237       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
 238       if (!CurrentIdx) {
 239         CurrentIdx = Default;
 240         break;
 241       }
 242       OnFailResumeAt.push_back(Default);
 243       break;
 244     }
 245
 246     case GIM_CheckNumOperands: {
 247       int64_t InsnID = MatchTable[CurrentIdx++];
 248       int64_t Expected = MatchTable[CurrentIdx++];
 249       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 250                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
 251                              << InsnID << "], Expected=" << Expected << ")\n");
 252       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 253       if (State.MIs[InsnID]->getNumOperands() != Expected) {
 254         if (handleReject() == RejectAndGiveUp)
 255           return false;
 256       }
 257       break;
 258     }
 259     case GIM_CheckI64ImmPredicate: {
 260       int64_t InsnID = MatchTable[CurrentIdx++];
 261       int64_t Predicate = MatchTable[CurrentIdx++];
 262       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 263                       dbgs()
 264                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
 265                           << InsnID << "], Predicate=" << Predicate << ")\n");
 266       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 267       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 268              "Expected G_CONSTANT");
 269       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
 270       int64_t Value = 0;
 271       if (State.MIs[InsnID]->getOperand(1).isCImm())
 272         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
 273       else if (State.MIs[InsnID]->getOperand(1).isImm())
 274         Value = State.MIs[InsnID]->getOperand(1).getImm();
 275       else
 276         llvm_unreachable("Expected Imm or CImm operand");
 277
 278       if (!testImmPredicate_I64(Predicate, Value))
 279         if (handleReject() == RejectAndGiveUp)
 280           return false;
 281       break;
 282     }
 283     case GIM_CheckAPIntImmPredicate: {
 284       int64_t InsnID = MatchTable[CurrentIdx++];
 285       int64_t Predicate = MatchTable[CurrentIdx++];
 286       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 287                       dbgs()
 288                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
 289                           << InsnID << "], Predicate=" << Predicate << ")\n");
 290       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 291       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 292              "Expected G_CONSTANT");
 293       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
 294       APInt Value;
 295       if (State.MIs[InsnID]->getOperand(1).isCImm())
 296         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
 297       else
 298         llvm_unreachable("Expected Imm or CImm operand");
 299
 300       if (!testImmPredicate_APInt(Predicate, Value))
 301         if (handleReject() == RejectAndGiveUp)
 302           return false;
 303       break;
 304     }
 305     case GIM_CheckAPFloatImmPredicate: {
 306       int64_t InsnID = MatchTable[CurrentIdx++];
 307       int64_t Predicate = MatchTable[CurrentIdx++];
 308       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 309                       dbgs()
 310                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
 311                           << InsnID << "], Predicate=" << Predicate << ")\n");
 312       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 313       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
 314              "Expected G_FCONSTANT");
 315       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
 316       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
 317       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
 318
 319       if (!testImmPredicate_APFloat(Predicate, Value))
 320         if (handleReject() == RejectAndGiveUp)
 321           return false;
 322       break;
 323     }
 324     case GIM_CheckCxxInsnPredicate: {
 325       int64_t InsnID = MatchTable[CurrentIdx++];
 326       int64_t Predicate = MatchTable[CurrentIdx++];
 327       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 328                       dbgs()
 329                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
 330                           << InsnID << "], Predicate=" << Predicate << ")\n");
 331       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 332       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
 333
 334       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
 335         if (handleReject() == RejectAndGiveUp)
 336           return false;
 337       break;
 338     }
 339     case GIM_CheckAtomicOrdering: {
 340       int64_t InsnID = MatchTable[CurrentIdx++];
 341       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 342       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 343                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
 344                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 345       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 346       if (!State.MIs[InsnID]->hasOneMemOperand())
 347         if (handleReject() == RejectAndGiveUp)
 348           return false;
 349
 350       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 351         if (MMO->getOrdering() != Ordering)
 352           if (handleReject() == RejectAndGiveUp)
 353             return false;
 354       break;
 355     }
 356     case GIM_CheckAtomicOrderingOrStrongerThan: {
 357       int64_t InsnID = MatchTable[CurrentIdx++];
 358       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 359       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 360                       dbgs() << CurrentIdx
 361                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
 362                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 363       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 364       if (!State.MIs[InsnID]->hasOneMemOperand())
 365         if (handleReject() == RejectAndGiveUp)
 366           return false;
 367
 368       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 369         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
 370           if (handleReject() == RejectAndGiveUp)
 371             return false;
 372       break;
 373     }
 374     case GIM_CheckAtomicOrderingWeakerThan: {
 375       int64_t InsnID = MatchTable[CurrentIdx++];
 376       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 377       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 378                       dbgs() << CurrentIdx
 379                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
 380                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 381       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 382       if (!State.MIs[InsnID]->hasOneMemOperand())
 383         if (handleReject() == RejectAndGiveUp)
 384           return false;
 385
 386       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 387         if (!isStrongerThan(Ordering, MMO->getOrdering()))
 388           if (handleReject() == RejectAndGiveUp)
 389             return false;
 390       break;
 391     }
 392     case GIM_CheckMemoryAddressSpace: {
 393       int64_t InsnID = MatchTable[CurrentIdx++];
 394       int64_t MMOIdx = MatchTable[CurrentIdx++];
 395       // This accepts a list of possible address spaces.
 396       const int NumAddrSpace = MatchTable[CurrentIdx++];
 397
 398       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 399         if (handleReject() == RejectAndGiveUp)
 400           return false;
 401         break;
 402       }
 403
 404       // Need to still jump to the end of the list of address spaces if we find
 405       // a match earlier.
 406       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
 407
 408       const MachineMemOperand *MMO
 409         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 410       const unsigned MMOAddrSpace = MMO->getAddrSpace();
 411
 412       bool Success = false;
 413       for (int I = 0; I != NumAddrSpace; ++I) {
 414         unsigned AddrSpace = MatchTable[CurrentIdx++];
 415         DEBUG_WITH_TYPE(
 416           TgtInstructionSelector::getName(),
 417           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
 418                  << AddrSpace << '\n');
 419
 420         if (AddrSpace == MMOAddrSpace) {
 421           Success = true;
 422           break;
 423         }
 424       }
 425
 426       CurrentIdx = LastIdx;
 427       if (!Success && handleReject() == RejectAndGiveUp)
 428         return false;
 429       break;
 430     }
 431     case GIM_CheckMemoryAlignment: {
 432       int64_t InsnID = MatchTable[CurrentIdx++];
 433       int64_t MMOIdx = MatchTable[CurrentIdx++];
 434       unsigned MinAlign = MatchTable[CurrentIdx++];
 435
 436       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 437
 438       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 439         if (handleReject() == RejectAndGiveUp)
 440           return false;
 441         break;
 442       }
 443
 444       MachineMemOperand *MMO
 445         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 446       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 447                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
 448                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 449                       << ")->getAlignment() >= " << MinAlign << ")\n");
 450       if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
 451         return false;
 452
 453       break;
 454     }
 455     case GIM_CheckMemorySizeEqualTo: {
 456       int64_t InsnID = MatchTable[CurrentIdx++];
 457       int64_t MMOIdx = MatchTable[CurrentIdx++];
 458       uint64_t Size = MatchTable[CurrentIdx++];
 459
 460       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 461                       dbgs() << CurrentIdx
 462                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
 463                              << "]->memoperands() + " << MMOIdx
 464                              << ", Size=" << Size << ")\n");
 465       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 466
 467       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 468         if (handleReject() == RejectAndGiveUp)
 469           return false;
 470         break;
 471       }
 472
 473       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 474
 475       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 476                       dbgs() << MMO->getSize() << " bytes vs " << Size
 477                              << " bytes\n");
 478       if (MMO->getSize() != Size)
 479         if (handleReject() == RejectAndGiveUp)
 480           return false;
 481
 482       break;
 483     }
 484     case GIM_CheckMemorySizeEqualToLLT:
 485     case GIM_CheckMemorySizeLessThanLLT:
 486     case GIM_CheckMemorySizeGreaterThanLLT: {
 487       int64_t InsnID = MatchTable[CurrentIdx++];
 488       int64_t MMOIdx = MatchTable[CurrentIdx++];
 489       int64_t OpIdx = MatchTable[CurrentIdx++];
 490
 491       DEBUG_WITH_TYPE(
 492           TgtInstructionSelector::getName(),
 493           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
 494                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
 495                          ? "EqualTo"
 496                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
 497                                ? "GreaterThan"
 498                                : "LessThan")
 499                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 500                  << ", OpIdx=" << OpIdx << ")\n");
 501       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 502
 503       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 504       if (!MO.isReg()) {
 505         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 506                         dbgs() << CurrentIdx << ": Not a register\n");
 507         if (handleReject() == RejectAndGiveUp)
 508           return false;
 509         break;
 510       }
 511
 512       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 513         if (handleReject() == RejectAndGiveUp)
 514           return false;
 515         break;
 516       }
 517
 518       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 519
 520       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
 521       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
 522           MMO->getSizeInBits() != Size) {
 523         if (handleReject() == RejectAndGiveUp)
 524           return false;
 525       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
 526                  MMO->getSizeInBits() >= Size) {
 527         if (handleReject() == RejectAndGiveUp)
 528           return false;
 529       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
 530                  MMO->getSizeInBits() <= Size)
 531         if (handleReject() == RejectAndGiveUp)
 532           return false;
 533
 534       break;
 535     }
 536     case GIM_CheckType: {
 537       int64_t InsnID = MatchTable[CurrentIdx++];
 538       int64_t OpIdx = MatchTable[CurrentIdx++];
 539       int64_t TypeID = MatchTable[CurrentIdx++];
 540       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 541                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
 542                              << "]->getOperand(" << OpIdx
 543                              << "), TypeID=" << TypeID << ")\n");
 544       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 545       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 546       if (!MO.isReg() ||
 547           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
 548         if (handleReject() == RejectAndGiveUp)
 549           return false;
 550       }
 551       break;
 552     }
 553     case GIM_CheckPointerToAny: {
 554       int64_t InsnID = MatchTable[CurrentIdx++];
 555       int64_t OpIdx = MatchTable[CurrentIdx++];
 556       int64_t SizeInBits = MatchTable[CurrentIdx++];
 557
 558       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 559                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
 560                              << InsnID << "]->getOperand(" << OpIdx
 561                              << "), SizeInBits=" << SizeInBits << ")\n");
 562       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 563       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 564       const LLT Ty = MRI.getType(MO.getReg());
 565
 566       // iPTR must be looked up in the target.
 567       if (SizeInBits == 0) {
 568         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
 569         const unsigned AddrSpace = Ty.getAddressSpace();
 570         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
 571       }
 572
 573       assert(SizeInBits != 0 && "Pointer size must be known");
 574
 575       if (MO.isReg()) {
 576         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
 577           if (handleReject() == RejectAndGiveUp)
 578             return false;
 579       } else if (handleReject() == RejectAndGiveUp)
 580         return false;
 581
 582       break;
 583     }
 584     case GIM_CheckRegBankForClass: {
 585       int64_t InsnID = MatchTable[CurrentIdx++];
 586       int64_t OpIdx = MatchTable[CurrentIdx++];
 587       int64_t RCEnum = MatchTable[CurrentIdx++];
 588       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 589                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
 590                              << InsnID << "]->getOperand(" << OpIdx
 591                              << "), RCEnum=" << RCEnum << ")\n");
 592       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 593       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 594       if (!MO.isReg() ||
 595           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
 596                                       MRI.getType(MO.getReg())) !=
 597               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
 598         if (handleReject() == RejectAndGiveUp)
 599           return false;
 600       }
 601       break;
 602     }
 603
 604     case GIM_CheckComplexPattern: {
 605       int64_t InsnID = MatchTable[CurrentIdx++];
 606       int64_t OpIdx = MatchTable[CurrentIdx++];
 607       int64_t RendererID = MatchTable[CurrentIdx++];
 608       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
 609       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 610                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
 611                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
 612                              << "]->getOperand(" << OpIdx
 613                              << "), ComplexPredicateID=" << ComplexPredicateID
 614                              << ")\n");
 615       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 616       // FIXME: Use std::invoke() when it's available.
 617       ComplexRendererFns Renderer =
 618           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
 619               State.MIs[InsnID]->getOperand(OpIdx));
 620       if (Renderer.hasValue())
 621         State.Renderers[RendererID] = Renderer.getValue();
 622       else
 623         if (handleReject() == RejectAndGiveUp)
 624           return false;
 625       break;
 626     }
 627
 628     case GIM_CheckConstantInt: {
 629       int64_t InsnID = MatchTable[CurrentIdx++];
 630       int64_t OpIdx = MatchTable[CurrentIdx++];
 631       int64_t Value = MatchTable[CurrentIdx++];
 632       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 633                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
 634                              << InsnID << "]->getOperand(" << OpIdx
 635                              << "), Value=" << Value << ")\n");
 636       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 637       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 638       if (MO.isReg()) {
 639         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
 640         LLT Ty = MRI.getType(MO.getReg());
 641         Value = SignExtend64(Value, Ty.getSizeInBits());
 642
 643         if (!isOperandImmEqual(MO, Value, MRI)) {
 644           if (handleReject() == RejectAndGiveUp)
 645             return false;
 646         }
 647       } else if (handleReject() == RejectAndGiveUp)
 648         return false;
 649
 650       break;
 651     }
 652
 653     case GIM_CheckLiteralInt: {
 654       int64_t InsnID = MatchTable[CurrentIdx++];
 655       int64_t OpIdx = MatchTable[CurrentIdx++];
 656       int64_t Value = MatchTable[CurrentIdx++];
 657       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 658                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
 659                              << InsnID << "]->getOperand(" << OpIdx
 660                              << "), Value=" << Value << ")\n");
 661       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 662       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 663       if (MO.isImm() && MO.getImm() == Value)
 664         break;
 665
 666       if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
 667         break;
 668
 669       if (handleReject() == RejectAndGiveUp)
 670         return false;
 671
 672       break;
 673     }
 674
 675     case GIM_CheckIntrinsicID: {
 676       int64_t InsnID = MatchTable[CurrentIdx++];
 677       int64_t OpIdx = MatchTable[CurrentIdx++];
 678       int64_t Value = MatchTable[CurrentIdx++];
 679       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 680                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
 681                              << InsnID << "]->getOperand(" << OpIdx
 682                              << "), Value=" << Value << ")\n");
 683       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 684       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 685       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
 686         if (handleReject() == RejectAndGiveUp)
 687           return false;
 688       break;
 689     }
 690     case GIM_CheckCmpPredicate: {
 691       int64_t InsnID = MatchTable[CurrentIdx++];
 692       int64_t OpIdx = MatchTable[CurrentIdx++];
 693       int64_t Value = MatchTable[CurrentIdx++];
 694       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 695                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
 696                              << InsnID << "]->getOperand(" << OpIdx
 697                              << "), Value=" << Value << ")\n");
 698       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 699       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 700       if (!MO.isPredicate() || MO.getPredicate() != Value)
 701         if (handleReject() == RejectAndGiveUp)
 702           return false;
 703       break;
 704     }
 705     case GIM_CheckIsMBB: {
 706       int64_t InsnID = MatchTable[CurrentIdx++];
 707       int64_t OpIdx = MatchTable[CurrentIdx++];
 708       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 709                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
 710                              << "]->getOperand(" << OpIdx << "))\n");
 711       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 712       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
 713         if (handleReject() == RejectAndGiveUp)
 714           return false;
 715       }
 716       break;
 717     }
 718     case GIM_CheckIsImm: {
 719       int64_t InsnID = MatchTable[CurrentIdx++];
 720       int64_t OpIdx = MatchTable[CurrentIdx++];
 721       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 722                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
 723                              << "]->getOperand(" << OpIdx << "))\n");
 724       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 725       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
 726         if (handleReject() == RejectAndGiveUp)
 727           return false;
 728       }
 729       break;
 730     }
 731     case GIM_CheckIsSafeToFold: {
 732       int64_t InsnID = MatchTable[CurrentIdx++];
 733       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 734                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
 735                              << InsnID << "])\n");
 736       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 737       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
 738         if (handleReject() == RejectAndGiveUp)
 739           return false;
 740       }
 741       break;
 742     }
 743     case GIM_CheckIsSameOperand: {
 744       int64_t InsnID = MatchTable[CurrentIdx++];
 745       int64_t OpIdx = MatchTable[CurrentIdx++];
 746       int64_t OtherInsnID = MatchTable[CurrentIdx++];
 747       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
 748       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 749                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
 750                              << InsnID << "][" << OpIdx << "], MIs["
 751                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
 752       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 753       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
 754       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
 755               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
 756         if (handleReject() == RejectAndGiveUp)
 757           return false;
 758       }
 759       break;
 760     }
 761     case GIM_Reject:
 762       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 763                       dbgs() << CurrentIdx << ": GIM_Reject\n");
 764       if (handleReject() == RejectAndGiveUp)
 765         return false;
 766       break;
 767
 768     case GIR_MutateOpcode: {
 769       int64_t OldInsnID = MatchTable[CurrentIdx++];
 770       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 771       int64_t NewOpcode = MatchTable[CurrentIdx++];
 772       if (NewInsnID >= OutMIs.size())
 773         OutMIs.resize(NewInsnID + 1);
 774
 775       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
 776                                               State.MIs[OldInsnID]);
 777       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
 778       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 779                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
 780                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 781                              << NewOpcode << ")\n");
 782       break;
 783     }
 784
 785     case GIR_BuildMI: {
 786       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 787       int64_t Opcode = MatchTable[CurrentIdx++];
 788       if (NewInsnID >= OutMIs.size())
 789         OutMIs.resize(NewInsnID + 1);
 790
 791       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
 792                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
 793       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 794                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
 795                              << NewInsnID << "], " << Opcode << ")\n");
 796       break;
 797     }
 798
 799     case GIR_Copy: {
 800       int64_t NewInsnID = MatchTable[CurrentIdx++];
 801       int64_t OldInsnID = MatchTable[CurrentIdx++];
 802       int64_t OpIdx = MatchTable[CurrentIdx++];
 803       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 804       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
 805       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 806                       dbgs()
 807                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
 808                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
 809       break;
 810     }
 811
 812     case GIR_CopyOrAddZeroReg: {
 813       int64_t NewInsnID = MatchTable[CurrentIdx++];
 814       int64_t OldInsnID = MatchTable[CurrentIdx++];
 815       int64_t OpIdx = MatchTable[CurrentIdx++];
 816       int64_t ZeroReg = MatchTable[CurrentIdx++];
 817       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 818       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
 819       if (isOperandImmEqual(MO, 0, MRI))
 820         OutMIs[NewInsnID].addReg(ZeroReg);
 821       else
 822         OutMIs[NewInsnID].add(MO);
 823       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 824                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
 825                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 826                              << OpIdx << ", " << ZeroReg << ")\n");
 827       break;
 828     }
 829
 830     case GIR_CopySubReg: {
 831       int64_t NewInsnID = MatchTable[CurrentIdx++];
 832       int64_t OldInsnID = MatchTable[CurrentIdx++];
 833       int64_t OpIdx = MatchTable[CurrentIdx++];
 834       int64_t SubRegIdx = MatchTable[CurrentIdx++];
 835       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 836       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
 837                                0, SubRegIdx);
 838       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 839                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
 840                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 841                              << OpIdx << ", " << SubRegIdx << ")\n");
 842       break;
 843     }
 844
 845     case GIR_AddImplicitDef: {
 846       int64_t InsnID = MatchTable[CurrentIdx++];
 847       int64_t RegNum = MatchTable[CurrentIdx++];
 848       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 849       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
 850       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 851                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
 852                              << InsnID << "], " << RegNum << ")\n");
 853       break;
 854     }
 855
 856     case GIR_AddImplicitUse: {
 857       int64_t InsnID = MatchTable[CurrentIdx++];
 858       int64_t RegNum = MatchTable[CurrentIdx++];
 859       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 860       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
 861       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 862                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
 863                              << InsnID << "], " << RegNum << ")\n");
 864       break;
 865     }
 866
 867     case GIR_AddRegister: {
 868       int64_t InsnID = MatchTable[CurrentIdx++];
 869       int64_t RegNum = MatchTable[CurrentIdx++];
 870       uint64_t RegFlags = MatchTable[CurrentIdx++];
 871       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 872       OutMIs[InsnID].addReg(RegNum, RegFlags);
 873       DEBUG_WITH_TYPE(
 874         TgtInstructionSelector::getName(),
 875         dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
 876         << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
 877       break;
 878     }
 879
 880     case GIR_AddTempRegister:
 881     case GIR_AddTempSubRegister: {
 882       int64_t InsnID = MatchTable[CurrentIdx++];
 883       int64_t TempRegID = MatchTable[CurrentIdx++];
 884       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
 885       unsigned SubReg = 0;
 886       if (MatcherOpcode == GIR_AddTempSubRegister)
 887         SubReg = MatchTable[CurrentIdx++];
 888
 889       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 890
 891       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
 892       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 893                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
 894                              << InsnID << "], TempRegisters[" << TempRegID
 895                              << "]";
 896                       if (SubReg)
 897                         dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
 898                       dbgs() << ", " << TempRegFlags << ")\n");
 899       break;
 900     }
 901
 902     case GIR_AddImm: {
 903       int64_t InsnID = MatchTable[CurrentIdx++];
 904       int64_t Imm = MatchTable[CurrentIdx++];
 905       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 906       OutMIs[InsnID].addImm(Imm);
 907       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 908                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
 909                              << "], " << Imm << ")\n");
 910       break;
 911     }
 912
 913     case GIR_ComplexRenderer: {
 914       int64_t InsnID = MatchTable[CurrentIdx++];
 915       int64_t RendererID = MatchTable[CurrentIdx++];
 916       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 917       for (const auto &RenderOpFn : State.Renderers[RendererID])
 918         RenderOpFn(OutMIs[InsnID]);
 919       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 920                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
 921                              << InsnID << "], " << RendererID << ")\n");
 922       break;
 923     }
 924     case GIR_ComplexSubOperandRenderer: {
 925       int64_t InsnID = MatchTable[CurrentIdx++];
 926       int64_t RendererID = MatchTable[CurrentIdx++];
 927       int64_t RenderOpID = MatchTable[CurrentIdx++];
 928       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 929       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
 930       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 931                       dbgs() << CurrentIdx
 932                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
 933                              << InsnID << "], " << RendererID << ", "
 934                              << RenderOpID << ")\n");
 935       break;
 936     }
 937
 938     case GIR_CopyConstantAsSImm: {
 939       int64_t NewInsnID = MatchTable[CurrentIdx++];
 940       int64_t OldInsnID = MatchTable[CurrentIdx++];
 941       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 942       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
 943       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
 944         OutMIs[NewInsnID].addImm(
 945             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
 946       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
 947         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
 948       else
 949         llvm_unreachable("Expected Imm or CImm operand");
 950       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 951                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
 952                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 953       break;
 954     }
 955
 956     // TODO: Needs a test case once we have a pattern that uses this.
 957     case GIR_CopyFConstantAsFPImm: {
 958       int64_t NewInsnID = MatchTable[CurrentIdx++];
 959       int64_t OldInsnID = MatchTable[CurrentIdx++];
 960       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 961       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
 962       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
 963         OutMIs[NewInsnID].addFPImm(
 964             State.MIs[OldInsnID]->getOperand(1).getFPImm());
 965       else
 966         llvm_unreachable("Expected FPImm operand");
 967       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 968                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
 969                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 970       break;
 971     }
 972
 973     case GIR_CustomRenderer: {
 974       int64_t InsnID = MatchTable[CurrentIdx++];
 975       int64_t OldInsnID = MatchTable[CurrentIdx++];
 976       int64_t RendererFnID = MatchTable[CurrentIdx++];
 977       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 978       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 979                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
 980                              << InsnID << "], MIs[" << OldInsnID << "], "
 981                              << RendererFnID << ")\n");
 982       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
 983         OutMIs[InsnID], *State.MIs[OldInsnID],
 984         -1); // Not a source operand of the old instruction.
 985       break;
 986     }
 987     case GIR_CustomOperandRenderer: {
 988       int64_t InsnID = MatchTable[CurrentIdx++];
 989       int64_t OldInsnID = MatchTable[CurrentIdx++];
 990       int64_t OpIdx = MatchTable[CurrentIdx++];
 991       int64_t RendererFnID = MatchTable[CurrentIdx++];
 992       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 993
 994       DEBUG_WITH_TYPE(
 995         TgtInstructionSelector::getName(),
 996         dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
 997                << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
 998                << OpIdx << "), "
 999         << RendererFnID << ")\n");
1000       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
1001                                                      *State.MIs[OldInsnID],
1002                                                      OpIdx);
1003       break;
1004     }
1005     case GIR_ConstrainOperandRC: {
1006       int64_t InsnID = MatchTable[CurrentIdx++];
1007       int64_t OpIdx = MatchTable[CurrentIdx++];
1008       int64_t RCEnum = MatchTable[CurrentIdx++];
1009       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1010       constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
1011                                     *TRI.getRegClass(RCEnum), TII, TRI, RBI);
1012       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1013                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1014                              << InsnID << "], " << OpIdx << ", " << RCEnum
1015                              << ")\n");
1016       break;
1017     }
1018
1019     case GIR_ConstrainSelectedInstOperands: {
1020       int64_t InsnID = MatchTable[CurrentIdx++];
1021       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1022       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1023                                        RBI);
1024       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1025                       dbgs() << CurrentIdx
1026                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1027                              << InsnID << "])\n");
1028       break;
1029     }
1030
1031     case GIR_MergeMemOperands: {
1032       int64_t InsnID = MatchTable[CurrentIdx++];
1033       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1034
1035       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1036                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1037                              << InsnID << "]");
1038       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1039       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1040              GIU_MergeMemOperands_EndOfList) {
1041         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1042                         dbgs() << ", MIs[" << MergeInsnID << "]");
1043         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1044           OutMIs[InsnID].addMemOperand(MMO);
1045       }
1046       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
1047       break;
1048     }
1049
1050     case GIR_EraseFromParent: {
1051       int64_t InsnID = MatchTable[CurrentIdx++];
1052       assert(State.MIs[InsnID] &&
1053              "Attempted to erase an undefined instruction");
1054       State.MIs[InsnID]->eraseFromParent();
1055       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1056                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1057                              << InsnID << "])\n");
1058       break;
1059     }
1060
1061     case GIR_MakeTempReg: {
1062       int64_t TempRegID = MatchTable[CurrentIdx++];
1063       int64_t TypeID = MatchTable[CurrentIdx++];
1064
1065       State.TempRegisters[TempRegID] =
1066           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1067       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1068                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1069                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1070       break;
1071     }
1072
1073     case GIR_Coverage: {
1074       int64_t RuleID = MatchTable[CurrentIdx++];
1075       CoverageInfo.setCovered(RuleID);
1076
1077       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1078                       dbgs()
1079                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1080       break;
1081     }
1082
1083     case GIR_Done:
1084       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1085                       dbgs() << CurrentIdx << ": GIR_Done\n");
1086       propagateFlags(OutMIs);
1087       return true;
1088
1089     default:
1090       llvm_unreachable("Unexpected command");
1091     }
1092   }
1093 }
1094
1095 } // end namespace llvm
1096
1097 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H