contrib/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

   1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file declares the API for the instruction selector.
  10 /// This class is responsible for selecting machine instructions.
  11 /// It's implemented by the target. It's used by the InstructionSelect pass.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  17
  18 #include "llvm/ADT/SmallVector.h"
  19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  21 #include "llvm/CodeGen/GlobalISel/Utils.h"
  22 #include "llvm/CodeGen/MachineInstrBuilder.h"
  23 #include "llvm/CodeGen/MachineOperand.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetInstrInfo.h"
  26 #include "llvm/CodeGen/TargetOpcodes.h"
  27 #include "llvm/CodeGen/TargetRegisterInfo.h"
  28 #include "llvm/IR/Constants.h"
  29 #include "llvm/Support/Debug.h"
  30 #include "llvm/Support/ErrorHandling.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include <cassert>
  33 #include <cstddef>
  34 #include <cstdint>
  35
  36 namespace llvm {
  37
  38 /// GlobalISel PatFrag Predicates
  39 enum {
  40   GIPFP_I64_Invalid = 0,
  41   GIPFP_APInt_Invalid = 0,
  42   GIPFP_APFloat_Invalid = 0,
  43   GIPFP_MI_Invalid = 0,
  44 };
  45
  46 template <class TgtInstructionSelector, class PredicateBitset,
  47           class ComplexMatcherMemFn, class CustomRendererFn>
  48 bool InstructionSelector::executeMatchTable(
  49     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
  50     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
  51         &ISelInfo,
  52     const int64_t *MatchTable, const TargetInstrInfo &TII,
  53     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
  54     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
  55     CodeGenCoverage &CoverageInfo) const {
  56
  57   uint64_t CurrentIdx = 0;
  58   SmallVector<uint64_t, 4> OnFailResumeAt;
  59
  60   enum RejectAction { RejectAndGiveUp, RejectAndResume };
  61   auto handleReject = [&]() -> RejectAction {
  62     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  63                     dbgs() << CurrentIdx << ": Rejected\n");
  64     if (OnFailResumeAt.empty())
  65       return RejectAndGiveUp;
  66     CurrentIdx = OnFailResumeAt.pop_back_val();
  67     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  68                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
  69                            << OnFailResumeAt.size() << " try-blocks remain)\n");
  70     return RejectAndResume;
  71   };
  72
  73   while (true) {
  74     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
  75     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
  76     switch (MatcherOpcode) {
  77     case GIM_Try: {
  78       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  79                       dbgs() << CurrentIdx << ": Begin try-block\n");
  80       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
  81       break;
  82     }
  83
  84     case GIM_RecordInsn: {
  85       int64_t NewInsnID = MatchTable[CurrentIdx++];
  86       int64_t InsnID = MatchTable[CurrentIdx++];
  87       int64_t OpIdx = MatchTable[CurrentIdx++];
  88
  89       // As an optimisation we require that MIs[0] is always the root. Refuse
  90       // any attempt to modify it.
  91       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
  92
  93       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  94       if (!MO.isReg()) {
  95         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  96                         dbgs() << CurrentIdx << ": Not a register\n");
  97         if (handleReject() == RejectAndGiveUp)
  98           return false;
  99         break;
 100       }
 101       if (TRI.isPhysicalRegister(MO.getReg())) {
 102         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 103                         dbgs() << CurrentIdx << ": Is a physical register\n");
 104         if (handleReject() == RejectAndGiveUp)
 105           return false;
 106         break;
 107       }
 108
 109       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
 110       if ((size_t)NewInsnID < State.MIs.size())
 111         State.MIs[NewInsnID] = NewMI;
 112       else {
 113         assert((size_t)NewInsnID == State.MIs.size() &&
 114                "Expected to store MIs in order");
 115         State.MIs.push_back(NewMI);
 116       }
 117       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 118                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
 119                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
 120                              << ")\n");
 121       break;
 122     }
 123
 124     case GIM_CheckFeatures: {
 125       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
 126       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 127                       dbgs() << CurrentIdx
 128                              << ": GIM_CheckFeatures(ExpectedBitsetID="
 129                              << ExpectedBitsetID << ")\n");
 130       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
 131           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
 132         if (handleReject() == RejectAndGiveUp)
 133           return false;
 134       }
 135       break;
 136     }
 137
 138     case GIM_CheckOpcode: {
 139       int64_t InsnID = MatchTable[CurrentIdx++];
 140       int64_t Expected = MatchTable[CurrentIdx++];
 141
 142       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 143       unsigned Opcode = State.MIs[InsnID]->getOpcode();
 144
 145       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 146                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
 147                              << "], ExpectedOpcode=" << Expected
 148                              << ") // Got=" << Opcode << "\n");
 149       if (Opcode != Expected) {
 150         if (handleReject() == RejectAndGiveUp)
 151           return false;
 152       }
 153       break;
 154     }
 155
 156     case GIM_SwitchOpcode: {
 157       int64_t InsnID = MatchTable[CurrentIdx++];
 158       int64_t LowerBound = MatchTable[CurrentIdx++];
 159       int64_t UpperBound = MatchTable[CurrentIdx++];
 160       int64_t Default = MatchTable[CurrentIdx++];
 161
 162       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 163       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
 164
 165       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 166         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
 167                << LowerBound << ", " << UpperBound << "), Default=" << Default
 168                << ", JumpTable...) // Got=" << Opcode << "\n";
 169       });
 170       if (Opcode < LowerBound || UpperBound <= Opcode) {
 171         CurrentIdx = Default;
 172         break;
 173       }
 174       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
 175       if (!CurrentIdx) {
 176         CurrentIdx = Default;
 177         break;
 178       }
 179       OnFailResumeAt.push_back(Default);
 180       break;
 181     }
 182
 183     case GIM_SwitchType: {
 184       int64_t InsnID = MatchTable[CurrentIdx++];
 185       int64_t OpIdx = MatchTable[CurrentIdx++];
 186       int64_t LowerBound = MatchTable[CurrentIdx++];
 187       int64_t UpperBound = MatchTable[CurrentIdx++];
 188       int64_t Default = MatchTable[CurrentIdx++];
 189
 190       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 191       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 192
 193       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 194         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
 195                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
 196                << UpperBound << "), Default=" << Default
 197                << ", JumpTable...) // Got=";
 198         if (!MO.isReg())
 199           dbgs() << "Not a VReg\n";
 200         else
 201           dbgs() << MRI.getType(MO.getReg()) << "\n";
 202       });
 203       if (!MO.isReg()) {
 204         CurrentIdx = Default;
 205         break;
 206       }
 207       const LLT Ty = MRI.getType(MO.getReg());
 208       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
 209       if (TyI == ISelInfo.TypeIDMap.end()) {
 210         CurrentIdx = Default;
 211         break;
 212       }
 213       const int64_t TypeID = TyI->second;
 214       if (TypeID < LowerBound || UpperBound <= TypeID) {
 215         CurrentIdx = Default;
 216         break;
 217       }
 218       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
 219       if (!CurrentIdx) {
 220         CurrentIdx = Default;
 221         break;
 222       }
 223       OnFailResumeAt.push_back(Default);
 224       break;
 225     }
 226
 227     case GIM_CheckNumOperands: {
 228       int64_t InsnID = MatchTable[CurrentIdx++];
 229       int64_t Expected = MatchTable[CurrentIdx++];
 230       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 231                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
 232                              << InsnID << "], Expected=" << Expected << ")\n");
 233       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 234       if (State.MIs[InsnID]->getNumOperands() != Expected) {
 235         if (handleReject() == RejectAndGiveUp)
 236           return false;
 237       }
 238       break;
 239     }
 240     case GIM_CheckI64ImmPredicate: {
 241       int64_t InsnID = MatchTable[CurrentIdx++];
 242       int64_t Predicate = MatchTable[CurrentIdx++];
 243       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 244                       dbgs()
 245                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
 246                           << InsnID << "], Predicate=" << Predicate << ")\n");
 247       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 248       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 249              "Expected G_CONSTANT");
 250       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
 251       int64_t Value = 0;
 252       if (State.MIs[InsnID]->getOperand(1).isCImm())
 253         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
 254       else if (State.MIs[InsnID]->getOperand(1).isImm())
 255         Value = State.MIs[InsnID]->getOperand(1).getImm();
 256       else
 257         llvm_unreachable("Expected Imm or CImm operand");
 258
 259       if (!testImmPredicate_I64(Predicate, Value))
 260         if (handleReject() == RejectAndGiveUp)
 261           return false;
 262       break;
 263     }
 264     case GIM_CheckAPIntImmPredicate: {
 265       int64_t InsnID = MatchTable[CurrentIdx++];
 266       int64_t Predicate = MatchTable[CurrentIdx++];
 267       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 268                       dbgs()
 269                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
 270                           << InsnID << "], Predicate=" << Predicate << ")\n");
 271       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 272       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 273              "Expected G_CONSTANT");
 274       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
 275       APInt Value;
 276       if (State.MIs[InsnID]->getOperand(1).isCImm())
 277         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
 278       else
 279         llvm_unreachable("Expected Imm or CImm operand");
 280
 281       if (!testImmPredicate_APInt(Predicate, Value))
 282         if (handleReject() == RejectAndGiveUp)
 283           return false;
 284       break;
 285     }
 286     case GIM_CheckAPFloatImmPredicate: {
 287       int64_t InsnID = MatchTable[CurrentIdx++];
 288       int64_t Predicate = MatchTable[CurrentIdx++];
 289       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 290                       dbgs()
 291                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
 292                           << InsnID << "], Predicate=" << Predicate << ")\n");
 293       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 294       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
 295              "Expected G_FCONSTANT");
 296       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
 297       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
 298       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
 299
 300       if (!testImmPredicate_APFloat(Predicate, Value))
 301         if (handleReject() == RejectAndGiveUp)
 302           return false;
 303       break;
 304     }
 305     case GIM_CheckCxxInsnPredicate: {
 306       int64_t InsnID = MatchTable[CurrentIdx++];
 307       int64_t Predicate = MatchTable[CurrentIdx++];
 308       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 309                       dbgs()
 310                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
 311                           << InsnID << "], Predicate=" << Predicate << ")\n");
 312       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 313       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
 314
 315       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
 316         if (handleReject() == RejectAndGiveUp)
 317           return false;
 318       break;
 319     }
 320     case GIM_CheckAtomicOrdering: {
 321       int64_t InsnID = MatchTable[CurrentIdx++];
 322       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 323       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 324                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
 325                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 326       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 327       if (!State.MIs[InsnID]->hasOneMemOperand())
 328         if (handleReject() == RejectAndGiveUp)
 329           return false;
 330
 331       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 332         if (MMO->getOrdering() != Ordering)
 333           if (handleReject() == RejectAndGiveUp)
 334             return false;
 335       break;
 336     }
 337     case GIM_CheckAtomicOrderingOrStrongerThan: {
 338       int64_t InsnID = MatchTable[CurrentIdx++];
 339       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 340       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 341                       dbgs() << CurrentIdx
 342                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
 343                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 344       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 345       if (!State.MIs[InsnID]->hasOneMemOperand())
 346         if (handleReject() == RejectAndGiveUp)
 347           return false;
 348
 349       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 350         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
 351           if (handleReject() == RejectAndGiveUp)
 352             return false;
 353       break;
 354     }
 355     case GIM_CheckAtomicOrderingWeakerThan: {
 356       int64_t InsnID = MatchTable[CurrentIdx++];
 357       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 358       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 359                       dbgs() << CurrentIdx
 360                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
 361                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 362       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 363       if (!State.MIs[InsnID]->hasOneMemOperand())
 364         if (handleReject() == RejectAndGiveUp)
 365           return false;
 366
 367       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 368         if (!isStrongerThan(Ordering, MMO->getOrdering()))
 369           if (handleReject() == RejectAndGiveUp)
 370             return false;
 371       break;
 372     }
 373     case GIM_CheckMemoryAddressSpace: {
 374       int64_t InsnID = MatchTable[CurrentIdx++];
 375       int64_t MMOIdx = MatchTable[CurrentIdx++];
 376       // This accepts a list of possible address spaces.
 377       const int NumAddrSpace = MatchTable[CurrentIdx++];
 378
 379       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 380         if (handleReject() == RejectAndGiveUp)
 381           return false;
 382         break;
 383       }
 384
 385       // Need to still jump to the end of the list of address spaces if we find
 386       // a match earlier.
 387       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
 388
 389       const MachineMemOperand *MMO
 390         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 391       const unsigned MMOAddrSpace = MMO->getAddrSpace();
 392
 393       bool Success = false;
 394       for (int I = 0; I != NumAddrSpace; ++I) {
 395         unsigned AddrSpace = MatchTable[CurrentIdx++];
 396         DEBUG_WITH_TYPE(
 397           TgtInstructionSelector::getName(),
 398           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
 399                  << AddrSpace << '\n');
 400
 401         if (AddrSpace == MMOAddrSpace) {
 402           Success = true;
 403           break;
 404         }
 405       }
 406
 407       CurrentIdx = LastIdx;
 408       if (!Success && handleReject() == RejectAndGiveUp)
 409         return false;
 410       break;
 411     }
 412     case GIM_CheckMemorySizeEqualTo: {
 413       int64_t InsnID = MatchTable[CurrentIdx++];
 414       int64_t MMOIdx = MatchTable[CurrentIdx++];
 415       uint64_t Size = MatchTable[CurrentIdx++];
 416
 417       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 418                       dbgs() << CurrentIdx
 419                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
 420                              << "]->memoperands() + " << MMOIdx
 421                              << ", Size=" << Size << ")\n");
 422       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 423
 424       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 425         if (handleReject() == RejectAndGiveUp)
 426           return false;
 427         break;
 428       }
 429
 430       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 431
 432       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 433                       dbgs() << MMO->getSize() << " bytes vs " << Size
 434                              << " bytes\n");
 435       if (MMO->getSize() != Size)
 436         if (handleReject() == RejectAndGiveUp)
 437           return false;
 438
 439       break;
 440     }
 441     case GIM_CheckMemorySizeEqualToLLT:
 442     case GIM_CheckMemorySizeLessThanLLT:
 443     case GIM_CheckMemorySizeGreaterThanLLT: {
 444       int64_t InsnID = MatchTable[CurrentIdx++];
 445       int64_t MMOIdx = MatchTable[CurrentIdx++];
 446       int64_t OpIdx = MatchTable[CurrentIdx++];
 447
 448       DEBUG_WITH_TYPE(
 449           TgtInstructionSelector::getName(),
 450           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
 451                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
 452                          ? "EqualTo"
 453                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
 454                                ? "GreaterThan"
 455                                : "LessThan")
 456                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 457                  << ", OpIdx=" << OpIdx << ")\n");
 458       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 459
 460       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 461       if (!MO.isReg()) {
 462         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 463                         dbgs() << CurrentIdx << ": Not a register\n");
 464         if (handleReject() == RejectAndGiveUp)
 465           return false;
 466         break;
 467       }
 468
 469       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 470         if (handleReject() == RejectAndGiveUp)
 471           return false;
 472         break;
 473       }
 474
 475       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 476
 477       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
 478       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
 479           MMO->getSizeInBits() != Size) {
 480         if (handleReject() == RejectAndGiveUp)
 481           return false;
 482       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
 483                  MMO->getSizeInBits() >= Size) {
 484         if (handleReject() == RejectAndGiveUp)
 485           return false;
 486       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
 487                  MMO->getSizeInBits() <= Size)
 488         if (handleReject() == RejectAndGiveUp)
 489           return false;
 490
 491       break;
 492     }
 493     case GIM_CheckType: {
 494       int64_t InsnID = MatchTable[CurrentIdx++];
 495       int64_t OpIdx = MatchTable[CurrentIdx++];
 496       int64_t TypeID = MatchTable[CurrentIdx++];
 497       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 498                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
 499                              << "]->getOperand(" << OpIdx
 500                              << "), TypeID=" << TypeID << ")\n");
 501       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 502       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 503       if (!MO.isReg() ||
 504           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
 505         if (handleReject() == RejectAndGiveUp)
 506           return false;
 507       }
 508       break;
 509     }
 510     case GIM_CheckPointerToAny: {
 511       int64_t InsnID = MatchTable[CurrentIdx++];
 512       int64_t OpIdx = MatchTable[CurrentIdx++];
 513       int64_t SizeInBits = MatchTable[CurrentIdx++];
 514
 515       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 516                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
 517                              << InsnID << "]->getOperand(" << OpIdx
 518                              << "), SizeInBits=" << SizeInBits << ")\n");
 519       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 520       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 521       const LLT Ty = MRI.getType(MO.getReg());
 522
 523       // iPTR must be looked up in the target.
 524       if (SizeInBits == 0) {
 525         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
 526         const unsigned AddrSpace = Ty.getAddressSpace();
 527         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
 528       }
 529
 530       assert(SizeInBits != 0 && "Pointer size must be known");
 531
 532       if (MO.isReg()) {
 533         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
 534           if (handleReject() == RejectAndGiveUp)
 535             return false;
 536       } else if (handleReject() == RejectAndGiveUp)
 537         return false;
 538
 539       break;
 540     }
 541     case GIM_CheckRegBankForClass: {
 542       int64_t InsnID = MatchTable[CurrentIdx++];
 543       int64_t OpIdx = MatchTable[CurrentIdx++];
 544       int64_t RCEnum = MatchTable[CurrentIdx++];
 545       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 546                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
 547                              << InsnID << "]->getOperand(" << OpIdx
 548                              << "), RCEnum=" << RCEnum << ")\n");
 549       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 550       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 551       if (!MO.isReg() ||
 552           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
 553               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
 554         if (handleReject() == RejectAndGiveUp)
 555           return false;
 556       }
 557       break;
 558     }
 559
 560     case GIM_CheckComplexPattern: {
 561       int64_t InsnID = MatchTable[CurrentIdx++];
 562       int64_t OpIdx = MatchTable[CurrentIdx++];
 563       int64_t RendererID = MatchTable[CurrentIdx++];
 564       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
 565       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 566                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
 567                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
 568                              << "]->getOperand(" << OpIdx
 569                              << "), ComplexPredicateID=" << ComplexPredicateID
 570                              << ")\n");
 571       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 572       // FIXME: Use std::invoke() when it's available.
 573       ComplexRendererFns Renderer =
 574           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
 575               State.MIs[InsnID]->getOperand(OpIdx));
 576       if (Renderer.hasValue())
 577         State.Renderers[RendererID] = Renderer.getValue();
 578       else
 579         if (handleReject() == RejectAndGiveUp)
 580           return false;
 581       break;
 582     }
 583
 584     case GIM_CheckConstantInt: {
 585       int64_t InsnID = MatchTable[CurrentIdx++];
 586       int64_t OpIdx = MatchTable[CurrentIdx++];
 587       int64_t Value = MatchTable[CurrentIdx++];
 588       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 589                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
 590                              << InsnID << "]->getOperand(" << OpIdx
 591                              << "), Value=" << Value << ")\n");
 592       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 593       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 594       if (MO.isReg()) {
 595         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
 596         LLT Ty = MRI.getType(MO.getReg());
 597         Value = SignExtend64(Value, Ty.getSizeInBits());
 598
 599         if (!isOperandImmEqual(MO, Value, MRI)) {
 600           if (handleReject() == RejectAndGiveUp)
 601             return false;
 602         }
 603       } else if (handleReject() == RejectAndGiveUp)
 604         return false;
 605
 606       break;
 607     }
 608
 609     case GIM_CheckLiteralInt: {
 610       int64_t InsnID = MatchTable[CurrentIdx++];
 611       int64_t OpIdx = MatchTable[CurrentIdx++];
 612       int64_t Value = MatchTable[CurrentIdx++];
 613       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 614                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
 615                              << InsnID << "]->getOperand(" << OpIdx
 616                              << "), Value=" << Value << ")\n");
 617       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 618       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 619       if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
 620         if (handleReject() == RejectAndGiveUp)
 621           return false;
 622       }
 623       break;
 624     }
 625
 626     case GIM_CheckIntrinsicID: {
 627       int64_t InsnID = MatchTable[CurrentIdx++];
 628       int64_t OpIdx = MatchTable[CurrentIdx++];
 629       int64_t Value = MatchTable[CurrentIdx++];
 630       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 631                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
 632                              << InsnID << "]->getOperand(" << OpIdx
 633                              << "), Value=" << Value << ")\n");
 634       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 635       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 636       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
 637         if (handleReject() == RejectAndGiveUp)
 638           return false;
 639       break;
 640     }
 641
 642     case GIM_CheckIsMBB: {
 643       int64_t InsnID = MatchTable[CurrentIdx++];
 644       int64_t OpIdx = MatchTable[CurrentIdx++];
 645       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 646                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
 647                              << "]->getOperand(" << OpIdx << "))\n");
 648       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 649       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
 650         if (handleReject() == RejectAndGiveUp)
 651           return false;
 652       }
 653       break;
 654     }
 655
 656     case GIM_CheckIsSafeToFold: {
 657       int64_t InsnID = MatchTable[CurrentIdx++];
 658       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 659                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
 660                              << InsnID << "])\n");
 661       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 662       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
 663         if (handleReject() == RejectAndGiveUp)
 664           return false;
 665       }
 666       break;
 667     }
 668     case GIM_CheckIsSameOperand: {
 669       int64_t InsnID = MatchTable[CurrentIdx++];
 670       int64_t OpIdx = MatchTable[CurrentIdx++];
 671       int64_t OtherInsnID = MatchTable[CurrentIdx++];
 672       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
 673       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 674                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
 675                              << InsnID << "][" << OpIdx << "], MIs["
 676                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
 677       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 678       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
 679       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
 680               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
 681         if (handleReject() == RejectAndGiveUp)
 682           return false;
 683       }
 684       break;
 685     }
 686     case GIM_Reject:
 687       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 688                       dbgs() << CurrentIdx << ": GIM_Reject\n");
 689       if (handleReject() == RejectAndGiveUp)
 690         return false;
 691       break;
 692
 693     case GIR_MutateOpcode: {
 694       int64_t OldInsnID = MatchTable[CurrentIdx++];
 695       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 696       int64_t NewOpcode = MatchTable[CurrentIdx++];
 697       if (NewInsnID >= OutMIs.size())
 698         OutMIs.resize(NewInsnID + 1);
 699
 700       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
 701                                               State.MIs[OldInsnID]);
 702       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
 703       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 704                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
 705                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 706                              << NewOpcode << ")\n");
 707       break;
 708     }
 709
 710     case GIR_BuildMI: {
 711       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 712       int64_t Opcode = MatchTable[CurrentIdx++];
 713       if (NewInsnID >= OutMIs.size())
 714         OutMIs.resize(NewInsnID + 1);
 715
 716       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
 717                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
 718       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 719                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
 720                              << NewInsnID << "], " << Opcode << ")\n");
 721       break;
 722     }
 723
 724     case GIR_Copy: {
 725       int64_t NewInsnID = MatchTable[CurrentIdx++];
 726       int64_t OldInsnID = MatchTable[CurrentIdx++];
 727       int64_t OpIdx = MatchTable[CurrentIdx++];
 728       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 729       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
 730       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 731                       dbgs()
 732                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
 733                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
 734       break;
 735     }
 736
 737     case GIR_CopyOrAddZeroReg: {
 738       int64_t NewInsnID = MatchTable[CurrentIdx++];
 739       int64_t OldInsnID = MatchTable[CurrentIdx++];
 740       int64_t OpIdx = MatchTable[CurrentIdx++];
 741       int64_t ZeroReg = MatchTable[CurrentIdx++];
 742       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 743       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
 744       if (isOperandImmEqual(MO, 0, MRI))
 745         OutMIs[NewInsnID].addReg(ZeroReg);
 746       else
 747         OutMIs[NewInsnID].add(MO);
 748       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 749                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
 750                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 751                              << OpIdx << ", " << ZeroReg << ")\n");
 752       break;
 753     }
 754
 755     case GIR_CopySubReg: {
 756       int64_t NewInsnID = MatchTable[CurrentIdx++];
 757       int64_t OldInsnID = MatchTable[CurrentIdx++];
 758       int64_t OpIdx = MatchTable[CurrentIdx++];
 759       int64_t SubRegIdx = MatchTable[CurrentIdx++];
 760       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 761       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
 762                                0, SubRegIdx);
 763       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 764                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
 765                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 766                              << OpIdx << ", " << SubRegIdx << ")\n");
 767       break;
 768     }
 769
 770     case GIR_AddImplicitDef: {
 771       int64_t InsnID = MatchTable[CurrentIdx++];
 772       int64_t RegNum = MatchTable[CurrentIdx++];
 773       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 774       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
 775       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 776                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
 777                              << InsnID << "], " << RegNum << ")\n");
 778       break;
 779     }
 780
 781     case GIR_AddImplicitUse: {
 782       int64_t InsnID = MatchTable[CurrentIdx++];
 783       int64_t RegNum = MatchTable[CurrentIdx++];
 784       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 785       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
 786       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 787                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
 788                              << InsnID << "], " << RegNum << ")\n");
 789       break;
 790     }
 791
 792     case GIR_AddRegister: {
 793       int64_t InsnID = MatchTable[CurrentIdx++];
 794       int64_t RegNum = MatchTable[CurrentIdx++];
 795       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 796       OutMIs[InsnID].addReg(RegNum);
 797       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 798                       dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
 799                              << InsnID << "], " << RegNum << ")\n");
 800       break;
 801     }
 802
 803     case GIR_AddTempRegister: {
 804       int64_t InsnID = MatchTable[CurrentIdx++];
 805       int64_t TempRegID = MatchTable[CurrentIdx++];
 806       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
 807       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 808       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
 809       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 810                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
 811                              << InsnID << "], TempRegisters[" << TempRegID
 812                              << "], " << TempRegFlags << ")\n");
 813       break;
 814     }
 815
 816     case GIR_AddImm: {
 817       int64_t InsnID = MatchTable[CurrentIdx++];
 818       int64_t Imm = MatchTable[CurrentIdx++];
 819       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 820       OutMIs[InsnID].addImm(Imm);
 821       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 822                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
 823                              << "], " << Imm << ")\n");
 824       break;
 825     }
 826
 827     case GIR_ComplexRenderer: {
 828       int64_t InsnID = MatchTable[CurrentIdx++];
 829       int64_t RendererID = MatchTable[CurrentIdx++];
 830       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 831       for (const auto &RenderOpFn : State.Renderers[RendererID])
 832         RenderOpFn(OutMIs[InsnID]);
 833       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 834                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
 835                              << InsnID << "], " << RendererID << ")\n");
 836       break;
 837     }
 838     case GIR_ComplexSubOperandRenderer: {
 839       int64_t InsnID = MatchTable[CurrentIdx++];
 840       int64_t RendererID = MatchTable[CurrentIdx++];
 841       int64_t RenderOpID = MatchTable[CurrentIdx++];
 842       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 843       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
 844       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 845                       dbgs() << CurrentIdx
 846                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
 847                              << InsnID << "], " << RendererID << ", "
 848                              << RenderOpID << ")\n");
 849       break;
 850     }
 851
 852     case GIR_CopyConstantAsSImm: {
 853       int64_t NewInsnID = MatchTable[CurrentIdx++];
 854       int64_t OldInsnID = MatchTable[CurrentIdx++];
 855       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 856       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
 857       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
 858         OutMIs[NewInsnID].addImm(
 859             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
 860       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
 861         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
 862       else
 863         llvm_unreachable("Expected Imm or CImm operand");
 864       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 865                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
 866                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 867       break;
 868     }
 869
 870     // TODO: Needs a test case once we have a pattern that uses this.
 871     case GIR_CopyFConstantAsFPImm: {
 872       int64_t NewInsnID = MatchTable[CurrentIdx++];
 873       int64_t OldInsnID = MatchTable[CurrentIdx++];
 874       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 875       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
 876       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
 877         OutMIs[NewInsnID].addFPImm(
 878             State.MIs[OldInsnID]->getOperand(1).getFPImm());
 879       else
 880         llvm_unreachable("Expected FPImm operand");
 881       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 882                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
 883                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 884       break;
 885     }
 886
 887     case GIR_CustomRenderer: {
 888       int64_t InsnID = MatchTable[CurrentIdx++];
 889       int64_t OldInsnID = MatchTable[CurrentIdx++];
 890       int64_t RendererFnID = MatchTable[CurrentIdx++];
 891       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 892       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 893                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
 894                              << InsnID << "], MIs[" << OldInsnID << "], "
 895                              << RendererFnID << ")\n");
 896       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
 897                                                      *State.MIs[OldInsnID]);
 898       break;
 899     }
 900     case GIR_ConstrainOperandRC: {
 901       int64_t InsnID = MatchTable[CurrentIdx++];
 902       int64_t OpIdx = MatchTable[CurrentIdx++];
 903       int64_t RCEnum = MatchTable[CurrentIdx++];
 904       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 905       constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
 906                                     *TRI.getRegClass(RCEnum), TII, TRI, RBI);
 907       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 908                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
 909                              << InsnID << "], " << OpIdx << ", " << RCEnum
 910                              << ")\n");
 911       break;
 912     }
 913
 914     case GIR_ConstrainSelectedInstOperands: {
 915       int64_t InsnID = MatchTable[CurrentIdx++];
 916       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 917       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
 918                                        RBI);
 919       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 920                       dbgs() << CurrentIdx
 921                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
 922                              << InsnID << "])\n");
 923       break;
 924     }
 925
 926     case GIR_MergeMemOperands: {
 927       int64_t InsnID = MatchTable[CurrentIdx++];
 928       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 929
 930       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 931                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
 932                              << InsnID << "]");
 933       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
 934       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
 935              GIU_MergeMemOperands_EndOfList) {
 936         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 937                         dbgs() << ", MIs[" << MergeInsnID << "]");
 938         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
 939           OutMIs[InsnID].addMemOperand(MMO);
 940       }
 941       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
 942       break;
 943     }
 944
 945     case GIR_EraseFromParent: {
 946       int64_t InsnID = MatchTable[CurrentIdx++];
 947       assert(State.MIs[InsnID] &&
 948              "Attempted to erase an undefined instruction");
 949       State.MIs[InsnID]->eraseFromParent();
 950       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 951                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
 952                              << InsnID << "])\n");
 953       break;
 954     }
 955
 956     case GIR_MakeTempReg: {
 957       int64_t TempRegID = MatchTable[CurrentIdx++];
 958       int64_t TypeID = MatchTable[CurrentIdx++];
 959
 960       State.TempRegisters[TempRegID] =
 961           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
 962       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 963                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
 964                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
 965       break;
 966     }
 967
 968     case GIR_Coverage: {
 969       int64_t RuleID = MatchTable[CurrentIdx++];
 970       CoverageInfo.setCovered(RuleID);
 971
 972       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 973                       dbgs()
 974                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
 975       break;
 976     }
 977
 978     case GIR_Done:
 979       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 980                       dbgs() << CurrentIdx << ": GIR_Done\n");
 981       return true;
 982
 983     default:
 984       llvm_unreachable("Unexpected command");
 985     }
 986   }
 987 }
 988
 989 } // end namespace llvm
 990
 991 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H