include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

   1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file This file declares the API for the instruction selector.
  11 /// This class is responsible for selecting machine instructions.
  12 /// It's implemented by the target. It's used by the InstructionSelect pass.
  13 //
  14 //===----------------------------------------------------------------------===//
  15
  16 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  17 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  18
  19 #include "llvm/ADT/SmallVector.h"
  20 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  21 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  22 #include "llvm/CodeGen/MachineInstrBuilder.h"
  23 #include "llvm/CodeGen/MachineOperand.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetInstrInfo.h"
  26 #include "llvm/CodeGen/TargetOpcodes.h"
  27 #include "llvm/CodeGen/TargetRegisterInfo.h"
  28 #include "llvm/IR/Constants.h"
  29 #include "llvm/Support/Debug.h"
  30 #include "llvm/Support/ErrorHandling.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include <cassert>
  33 #include <cstddef>
  34 #include <cstdint>
  35
  36 namespace llvm {
  37
  38 /// GlobalISel PatFrag Predicates
  39 enum {
  40   GIPFP_I64_Invalid = 0,
  41   GIPFP_APInt_Invalid = 0,
  42   GIPFP_APFloat_Invalid = 0,
  43 };
  44
  45 template <class TgtInstructionSelector, class PredicateBitset,
  46           class ComplexMatcherMemFn>
  47 bool InstructionSelector::executeMatchTable(
  48     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
  49     const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo,
  50     const int64_t *MatchTable, const TargetInstrInfo &TII,
  51     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
  52     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
  53     CodeGenCoverage &CoverageInfo) const {
  54   uint64_t CurrentIdx = 0;
  55   SmallVector<uint64_t, 8> OnFailResumeAt;
  56
  57   enum RejectAction { RejectAndGiveUp, RejectAndResume };
  58   auto handleReject = [&]() -> RejectAction {
  59     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  60                     dbgs() << CurrentIdx << ": Rejected\n");
  61     if (OnFailResumeAt.empty())
  62       return RejectAndGiveUp;
  63     CurrentIdx = OnFailResumeAt.back();
  64     OnFailResumeAt.pop_back();
  65     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  66                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
  67                            << OnFailResumeAt.size() << " try-blocks remain)\n");
  68     return RejectAndResume;
  69   };
  70
  71   while (true) {
  72     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
  73     switch (MatchTable[CurrentIdx++]) {
  74     case GIM_Try: {
  75       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  76                       dbgs() << CurrentIdx << ": Begin try-block\n");
  77       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
  78       break;
  79     }
  80
  81     case GIM_RecordInsn: {
  82       int64_t NewInsnID = MatchTable[CurrentIdx++];
  83       int64_t InsnID = MatchTable[CurrentIdx++];
  84       int64_t OpIdx = MatchTable[CurrentIdx++];
  85
  86       // As an optimisation we require that MIs[0] is always the root. Refuse
  87       // any attempt to modify it.
  88       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
  89
  90       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  91       if (!MO.isReg()) {
  92         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  93                         dbgs() << CurrentIdx << ": Not a register\n");
  94         if (handleReject() == RejectAndGiveUp)
  95           return false;
  96         break;
  97       }
  98       if (TRI.isPhysicalRegister(MO.getReg())) {
  99         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 100                         dbgs() << CurrentIdx << ": Is a physical register\n");
 101         if (handleReject() == RejectAndGiveUp)
 102           return false;
 103         break;
 104       }
 105
 106       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
 107       if ((size_t)NewInsnID < State.MIs.size())
 108         State.MIs[NewInsnID] = NewMI;
 109       else {
 110         assert((size_t)NewInsnID == State.MIs.size() &&
 111                "Expected to store MIs in order");
 112         State.MIs.push_back(NewMI);
 113       }
 114       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 115                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
 116                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
 117                              << ")\n");
 118       break;
 119     }
 120
 121     case GIM_CheckFeatures: {
 122       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
 123       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 124                       dbgs() << CurrentIdx
 125                              << ": GIM_CheckFeatures(ExpectedBitsetID="
 126                              << ExpectedBitsetID << ")\n");
 127       if ((AvailableFeatures & MatcherInfo.FeatureBitsets[ExpectedBitsetID]) !=
 128           MatcherInfo.FeatureBitsets[ExpectedBitsetID]) {
 129         if (handleReject() == RejectAndGiveUp)
 130           return false;
 131       }
 132       break;
 133     }
 134
 135     case GIM_CheckOpcode: {
 136       int64_t InsnID = MatchTable[CurrentIdx++];
 137       int64_t Expected = MatchTable[CurrentIdx++];
 138
 139       unsigned Opcode = State.MIs[InsnID]->getOpcode();
 140       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 141                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
 142                              << "], ExpectedOpcode=" << Expected
 143                              << ") // Got=" << Opcode << "\n");
 144       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 145       if (Opcode != Expected) {
 146         if (handleReject() == RejectAndGiveUp)
 147           return false;
 148       }
 149       break;
 150     }
 151
 152     case GIM_CheckNumOperands: {
 153       int64_t InsnID = MatchTable[CurrentIdx++];
 154       int64_t Expected = MatchTable[CurrentIdx++];
 155       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 156                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
 157                              << InsnID << "], Expected=" << Expected << ")\n");
 158       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 159       if (State.MIs[InsnID]->getNumOperands() != Expected) {
 160         if (handleReject() == RejectAndGiveUp)
 161           return false;
 162       }
 163       break;
 164     }
 165     case GIM_CheckI64ImmPredicate: {
 166       int64_t InsnID = MatchTable[CurrentIdx++];
 167       int64_t Predicate = MatchTable[CurrentIdx++];
 168       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 169                       dbgs()
 170                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
 171                           << InsnID << "], Predicate=" << Predicate << ")\n");
 172       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 173       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 174              "Expected G_CONSTANT");
 175       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
 176       int64_t Value = 0;
 177       if (State.MIs[InsnID]->getOperand(1).isCImm())
 178         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
 179       else if (State.MIs[InsnID]->getOperand(1).isImm())
 180         Value = State.MIs[InsnID]->getOperand(1).getImm();
 181       else
 182         llvm_unreachable("Expected Imm or CImm operand");
 183
 184       if (!testImmPredicate_I64(Predicate, Value))
 185         if (handleReject() == RejectAndGiveUp)
 186           return false;
 187       break;
 188     }
 189     case GIM_CheckAPIntImmPredicate: {
 190       int64_t InsnID = MatchTable[CurrentIdx++];
 191       int64_t Predicate = MatchTable[CurrentIdx++];
 192       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 193                       dbgs()
 194                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
 195                           << InsnID << "], Predicate=" << Predicate << ")\n");
 196       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 197       assert(State.MIs[InsnID]->getOpcode() && "Expected G_CONSTANT");
 198       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
 199       APInt Value;
 200       if (State.MIs[InsnID]->getOperand(1).isCImm())
 201         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
 202       else
 203         llvm_unreachable("Expected Imm or CImm operand");
 204
 205       if (!testImmPredicate_APInt(Predicate, Value))
 206         if (handleReject() == RejectAndGiveUp)
 207           return false;
 208       break;
 209     }
 210     case GIM_CheckAPFloatImmPredicate: {
 211       int64_t InsnID = MatchTable[CurrentIdx++];
 212       int64_t Predicate = MatchTable[CurrentIdx++];
 213       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 214                       dbgs()
 215                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
 216                           << InsnID << "], Predicate=" << Predicate << ")\n");
 217       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 218       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
 219              "Expected G_FCONSTANT");
 220       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
 221       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
 222       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
 223
 224       if (!testImmPredicate_APFloat(Predicate, Value))
 225         if (handleReject() == RejectAndGiveUp)
 226           return false;
 227       break;
 228     }
 229     case GIM_CheckAtomicOrdering: {
 230       int64_t InsnID = MatchTable[CurrentIdx++];
 231       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 232       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 233                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
 234                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 235       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 236
 237       if (!State.MIs[InsnID]->hasOneMemOperand())
 238         if (handleReject() == RejectAndGiveUp)
 239           return false;
 240
 241       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 242         if (MMO->getOrdering() != Ordering)
 243           if (handleReject() == RejectAndGiveUp)
 244             return false;
 245       break;
 246     }
 247     case GIM_CheckAtomicOrderingOrStrongerThan: {
 248       int64_t InsnID = MatchTable[CurrentIdx++];
 249       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 250       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 251                       dbgs() << CurrentIdx
 252                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
 253                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 254       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 255
 256       if (!State.MIs[InsnID]->hasOneMemOperand())
 257         if (handleReject() == RejectAndGiveUp)
 258           return false;
 259
 260       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 261         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
 262           if (handleReject() == RejectAndGiveUp)
 263             return false;
 264       break;
 265     }
 266     case GIM_CheckAtomicOrderingWeakerThan: {
 267       int64_t InsnID = MatchTable[CurrentIdx++];
 268       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 269       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 270                       dbgs() << CurrentIdx
 271                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
 272                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 273       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 274
 275       if (!State.MIs[InsnID]->hasOneMemOperand())
 276         if (handleReject() == RejectAndGiveUp)
 277           return false;
 278
 279       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 280         if (!isStrongerThan(Ordering, MMO->getOrdering()))
 281           if (handleReject() == RejectAndGiveUp)
 282             return false;
 283       break;
 284     }
 285     case GIM_CheckType: {
 286       int64_t InsnID = MatchTable[CurrentIdx++];
 287       int64_t OpIdx = MatchTable[CurrentIdx++];
 288       int64_t TypeID = MatchTable[CurrentIdx++];
 289       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 290                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
 291                              << "]->getOperand(" << OpIdx
 292                              << "), TypeID=" << TypeID << ")\n");
 293       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 294       if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) !=
 295           MatcherInfo.TypeObjects[TypeID]) {
 296         if (handleReject() == RejectAndGiveUp)
 297           return false;
 298       }
 299       break;
 300     }
 301     case GIM_CheckPointerToAny: {
 302       int64_t InsnID = MatchTable[CurrentIdx++];
 303       int64_t OpIdx = MatchTable[CurrentIdx++];
 304       int64_t SizeInBits = MatchTable[CurrentIdx++];
 305
 306       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 307                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
 308                              << InsnID << "]->getOperand(" << OpIdx
 309                              << "), SizeInBits=" << SizeInBits << ")\n");
 310       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 311
 312       // iPTR must be looked up in the target.
 313       if (SizeInBits == 0) {
 314         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
 315         SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
 316       }
 317
 318       assert(SizeInBits != 0 && "Pointer size must be known");
 319
 320       const LLT &Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg());
 321       if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) {
 322         if (handleReject() == RejectAndGiveUp)
 323           return false;
 324       }
 325       break;
 326     }
 327     case GIM_CheckRegBankForClass: {
 328       int64_t InsnID = MatchTable[CurrentIdx++];
 329       int64_t OpIdx = MatchTable[CurrentIdx++];
 330       int64_t RCEnum = MatchTable[CurrentIdx++];
 331       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 332                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
 333                              << InsnID << "]->getOperand(" << OpIdx
 334                              << "), RCEnum=" << RCEnum << ")\n");
 335       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 336       if (&RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
 337           RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI,
 338                          TRI)) {
 339         if (handleReject() == RejectAndGiveUp)
 340           return false;
 341       }
 342       break;
 343     }
 344
 345     case GIM_CheckComplexPattern: {
 346       int64_t InsnID = MatchTable[CurrentIdx++];
 347       int64_t OpIdx = MatchTable[CurrentIdx++];
 348       int64_t RendererID = MatchTable[CurrentIdx++];
 349       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
 350       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 351                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
 352                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
 353                              << "]->getOperand(" << OpIdx
 354                              << "), ComplexPredicateID=" << ComplexPredicateID
 355                              << ")\n");
 356       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 357       // FIXME: Use std::invoke() when it's available.
 358       ComplexRendererFns Renderer =
 359           (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])(
 360               State.MIs[InsnID]->getOperand(OpIdx));
 361       if (Renderer.hasValue())
 362         State.Renderers[RendererID] = Renderer.getValue();
 363       else
 364         if (handleReject() == RejectAndGiveUp)
 365           return false;
 366       break;
 367     }
 368
 369     case GIM_CheckConstantInt: {
 370       int64_t InsnID = MatchTable[CurrentIdx++];
 371       int64_t OpIdx = MatchTable[CurrentIdx++];
 372       int64_t Value = MatchTable[CurrentIdx++];
 373       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 374                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
 375                              << InsnID << "]->getOperand(" << OpIdx
 376                              << "), Value=" << Value << ")\n");
 377       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 378
 379       // isOperandImmEqual() will sign-extend to 64-bits, so should we.
 380       LLT Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg());
 381       Value = SignExtend64(Value, Ty.getSizeInBits());
 382
 383       if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value,
 384                              MRI)) {
 385         if (handleReject() == RejectAndGiveUp)
 386           return false;
 387       }
 388       break;
 389     }
 390
 391     case GIM_CheckLiteralInt: {
 392       int64_t InsnID = MatchTable[CurrentIdx++];
 393       int64_t OpIdx = MatchTable[CurrentIdx++];
 394       int64_t Value = MatchTable[CurrentIdx++];
 395       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 396                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
 397                              << InsnID << "]->getOperand(" << OpIdx
 398                              << "), Value=" << Value << ")\n");
 399       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 400       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 401       if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
 402         if (handleReject() == RejectAndGiveUp)
 403           return false;
 404       }
 405       break;
 406     }
 407
 408     case GIM_CheckIntrinsicID: {
 409       int64_t InsnID = MatchTable[CurrentIdx++];
 410       int64_t OpIdx = MatchTable[CurrentIdx++];
 411       int64_t Value = MatchTable[CurrentIdx++];
 412       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 413                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
 414                              << InsnID << "]->getOperand(" << OpIdx
 415                              << "), Value=" << Value << ")\n");
 416       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 417       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 418       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
 419         if (handleReject() == RejectAndGiveUp)
 420           return false;
 421       break;
 422     }
 423
 424     case GIM_CheckIsMBB: {
 425       int64_t InsnID = MatchTable[CurrentIdx++];
 426       int64_t OpIdx = MatchTable[CurrentIdx++];
 427       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 428                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
 429                              << "]->getOperand(" << OpIdx << "))\n");
 430       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 431       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
 432         if (handleReject() == RejectAndGiveUp)
 433           return false;
 434       }
 435       break;
 436     }
 437
 438     case GIM_CheckIsSafeToFold: {
 439       int64_t InsnID = MatchTable[CurrentIdx++];
 440       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 441                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
 442                              << InsnID << "])\n");
 443       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 444       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
 445         if (handleReject() == RejectAndGiveUp)
 446           return false;
 447       }
 448       break;
 449     }
 450     case GIM_CheckIsSameOperand: {
 451       int64_t InsnID = MatchTable[CurrentIdx++];
 452       int64_t OpIdx = MatchTable[CurrentIdx++];
 453       int64_t OtherInsnID = MatchTable[CurrentIdx++];
 454       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
 455       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 456                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
 457                              << InsnID << "][" << OpIdx << "], MIs["
 458                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
 459       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 460       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
 461       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
 462               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
 463         if (handleReject() == RejectAndGiveUp)
 464           return false;
 465       }
 466       break;
 467     }
 468     case GIM_Reject:
 469       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 470                       dbgs() << CurrentIdx << ": GIM_Reject");
 471       if (handleReject() == RejectAndGiveUp)
 472         return false;
 473       break;
 474
 475     case GIR_MutateOpcode: {
 476       int64_t OldInsnID = MatchTable[CurrentIdx++];
 477       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 478       int64_t NewOpcode = MatchTable[CurrentIdx++];
 479       if (NewInsnID >= OutMIs.size())
 480         OutMIs.resize(NewInsnID + 1);
 481
 482       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
 483                                               State.MIs[OldInsnID]);
 484       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
 485       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 486                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
 487                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 488                              << NewOpcode << ")\n");
 489       break;
 490     }
 491
 492     case GIR_BuildMI: {
 493       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 494       int64_t Opcode = MatchTable[CurrentIdx++];
 495       if (NewInsnID >= OutMIs.size())
 496         OutMIs.resize(NewInsnID + 1);
 497
 498       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
 499                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
 500       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 501                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
 502                              << NewInsnID << "], " << Opcode << ")\n");
 503       break;
 504     }
 505
 506     case GIR_Copy: {
 507       int64_t NewInsnID = MatchTable[CurrentIdx++];
 508       int64_t OldInsnID = MatchTable[CurrentIdx++];
 509       int64_t OpIdx = MatchTable[CurrentIdx++];
 510       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 511       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
 512       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 513                       dbgs()
 514                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
 515                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
 516       break;
 517     }
 518
 519     case GIR_CopyOrAddZeroReg: {
 520       int64_t NewInsnID = MatchTable[CurrentIdx++];
 521       int64_t OldInsnID = MatchTable[CurrentIdx++];
 522       int64_t OpIdx = MatchTable[CurrentIdx++];
 523       int64_t ZeroReg = MatchTable[CurrentIdx++];
 524       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 525       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
 526       if (isOperandImmEqual(MO, 0, MRI))
 527         OutMIs[NewInsnID].addReg(ZeroReg);
 528       else
 529         OutMIs[NewInsnID].add(MO);
 530       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 531                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
 532                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 533                              << OpIdx << ", " << ZeroReg << ")\n");
 534       break;
 535     }
 536
 537     case GIR_CopySubReg: {
 538       int64_t NewInsnID = MatchTable[CurrentIdx++];
 539       int64_t OldInsnID = MatchTable[CurrentIdx++];
 540       int64_t OpIdx = MatchTable[CurrentIdx++];
 541       int64_t SubRegIdx = MatchTable[CurrentIdx++];
 542       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 543       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
 544                                0, SubRegIdx);
 545       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 546                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
 547                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 548                              << OpIdx << ", " << SubRegIdx << ")\n");
 549       break;
 550     }
 551
 552     case GIR_AddImplicitDef: {
 553       int64_t InsnID = MatchTable[CurrentIdx++];
 554       int64_t RegNum = MatchTable[CurrentIdx++];
 555       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 556       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
 557       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 558                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
 559                              << InsnID << "], " << RegNum << ")\n");
 560       break;
 561     }
 562
 563     case GIR_AddImplicitUse: {
 564       int64_t InsnID = MatchTable[CurrentIdx++];
 565       int64_t RegNum = MatchTable[CurrentIdx++];
 566       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 567       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
 568       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 569                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
 570                              << InsnID << "], " << RegNum << ")\n");
 571       break;
 572     }
 573
 574     case GIR_AddRegister: {
 575       int64_t InsnID = MatchTable[CurrentIdx++];
 576       int64_t RegNum = MatchTable[CurrentIdx++];
 577       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 578       OutMIs[InsnID].addReg(RegNum);
 579       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 580                       dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
 581                              << InsnID << "], " << RegNum << ")\n");
 582       break;
 583     }
 584
 585     case GIR_AddTempRegister: {
 586       int64_t InsnID = MatchTable[CurrentIdx++];
 587       int64_t TempRegID = MatchTable[CurrentIdx++];
 588       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
 589       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 590       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
 591       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 592                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
 593                              << InsnID << "], TempRegisters[" << TempRegID
 594                              << "], " << TempRegFlags << ")\n");
 595       break;
 596     }
 597
 598     case GIR_AddImm: {
 599       int64_t InsnID = MatchTable[CurrentIdx++];
 600       int64_t Imm = MatchTable[CurrentIdx++];
 601       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 602       OutMIs[InsnID].addImm(Imm);
 603       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 604                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
 605                              << "], " << Imm << ")\n");
 606       break;
 607     }
 608
 609     case GIR_ComplexRenderer: {
 610       int64_t InsnID = MatchTable[CurrentIdx++];
 611       int64_t RendererID = MatchTable[CurrentIdx++];
 612       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 613       for (const auto &RenderOpFn : State.Renderers[RendererID])
 614         RenderOpFn(OutMIs[InsnID]);
 615       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 616                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
 617                              << InsnID << "], " << RendererID << ")\n");
 618       break;
 619     }
 620     case GIR_ComplexSubOperandRenderer: {
 621       int64_t InsnID = MatchTable[CurrentIdx++];
 622       int64_t RendererID = MatchTable[CurrentIdx++];
 623       int64_t RenderOpID = MatchTable[CurrentIdx++];
 624       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 625       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
 626       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 627                       dbgs() << CurrentIdx
 628                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
 629                              << InsnID << "], " << RendererID << ", "
 630                              << RenderOpID << ")\n");
 631       break;
 632     }
 633
 634     case GIR_CopyConstantAsSImm: {
 635       int64_t NewInsnID = MatchTable[CurrentIdx++];
 636       int64_t OldInsnID = MatchTable[CurrentIdx++];
 637       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 638       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
 639       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
 640         OutMIs[NewInsnID].addImm(
 641             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
 642       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
 643         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
 644       else
 645         llvm_unreachable("Expected Imm or CImm operand");
 646       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 647                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
 648                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 649       break;
 650     }
 651
 652     case GIR_ConstrainOperandRC: {
 653       int64_t InsnID = MatchTable[CurrentIdx++];
 654       int64_t OpIdx = MatchTable[CurrentIdx++];
 655       int64_t RCEnum = MatchTable[CurrentIdx++];
 656       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 657       constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
 658                                     *TRI.getRegClass(RCEnum), TII, TRI, RBI);
 659       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 660                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
 661                              << InsnID << "], " << OpIdx << ", " << RCEnum
 662                              << ")\n");
 663       break;
 664     }
 665
 666     case GIR_ConstrainSelectedInstOperands: {
 667       int64_t InsnID = MatchTable[CurrentIdx++];
 668       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 669       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
 670                                        RBI);
 671       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 672                       dbgs() << CurrentIdx
 673                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
 674                              << InsnID << "])\n");
 675       break;
 676     }
 677
 678     case GIR_MergeMemOperands: {
 679       int64_t InsnID = MatchTable[CurrentIdx++];
 680       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 681
 682       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 683                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
 684                              << InsnID << "]");
 685       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
 686       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
 687              GIU_MergeMemOperands_EndOfList) {
 688         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 689                         dbgs() << ", MIs[" << MergeInsnID << "]");
 690         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
 691           OutMIs[InsnID].addMemOperand(MMO);
 692       }
 693       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
 694       break;
 695     }
 696
 697     case GIR_EraseFromParent: {
 698       int64_t InsnID = MatchTable[CurrentIdx++];
 699       assert(State.MIs[InsnID] &&
 700              "Attempted to erase an undefined instruction");
 701       State.MIs[InsnID]->eraseFromParent();
 702       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 703                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
 704                              << InsnID << "])\n");
 705       break;
 706     }
 707
 708     case GIR_MakeTempReg: {
 709       int64_t TempRegID = MatchTable[CurrentIdx++];
 710       int64_t TypeID = MatchTable[CurrentIdx++];
 711
 712       State.TempRegisters[TempRegID] =
 713           MRI.createGenericVirtualRegister(MatcherInfo.TypeObjects[TypeID]);
 714       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 715                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
 716                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
 717       break;
 718     }
 719
 720     case GIR_Coverage: {
 721       int64_t RuleID = MatchTable[CurrentIdx++];
 722       CoverageInfo.setCovered(RuleID);
 723
 724       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 725                       dbgs()
 726                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
 727       break;
 728     }
 729
 730     case GIR_Done:
 731       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 732                       dbgs() << CurrentIdx << ": GIR_Done");
 733       return true;
 734
 735     default:
 736       llvm_unreachable("Unexpected command");
 737     }
 738   }
 739 }
 740
 741 } // end namespace llvm
 742
 743 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H