1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 /// \file This file declares the API for the instruction selector.
10 /// This class is responsible for selecting machine instructions.
11 /// It's implemented by the target. It's used by the InstructionSelect pass.
13 //===----------------------------------------------------------------------===//
15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
21 #include "llvm/CodeGen/GlobalISel/Utils.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/CodeGen/TargetOpcodes.h"
27 #include "llvm/CodeGen/TargetRegisterInfo.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/Support/Debug.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/raw_ostream.h"
38 /// GlobalISel PatFrag Predicates
40 GIPFP_I64_Invalid = 0,
41 GIPFP_APInt_Invalid = 0,
42 GIPFP_APFloat_Invalid = 0,
46 template <class TgtInstructionSelector, class PredicateBitset,
47 class ComplexMatcherMemFn, class CustomRendererFn>
48 bool InstructionSelector::executeMatchTable(
49 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
50 const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52 const int64_t *MatchTable, const TargetInstrInfo &TII,
53 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
54 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
55 CodeGenCoverage &CoverageInfo) const {
57 uint64_t CurrentIdx = 0;
58 SmallVector<uint64_t, 4> OnFailResumeAt;
60 enum RejectAction { RejectAndGiveUp, RejectAndResume };
61 auto handleReject = [&]() -> RejectAction {
62 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63 dbgs() << CurrentIdx << ": Rejected\n");
64 if (OnFailResumeAt.empty())
65 return RejectAndGiveUp;
66 CurrentIdx = OnFailResumeAt.pop_back_val();
67 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69 << OnFailResumeAt.size() << " try-blocks remain)\n");
70 return RejectAndResume;
74 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75 int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76 switch (MatcherOpcode) {
78 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79 dbgs() << CurrentIdx << ": Begin try-block\n");
80 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
84 case GIM_RecordInsn: {
85 int64_t NewInsnID = MatchTable[CurrentIdx++];
86 int64_t InsnID = MatchTable[CurrentIdx++];
87 int64_t OpIdx = MatchTable[CurrentIdx++];
89 // As an optimisation we require that MIs[0] is always the root. Refuse
90 // any attempt to modify it.
91 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96 dbgs() << CurrentIdx << ": Not a register\n");
97 if (handleReject() == RejectAndGiveUp)
101 if (TRI.isPhysicalRegister(MO.getReg())) {
102 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103 dbgs() << CurrentIdx << ": Is a physical register\n");
104 if (handleReject() == RejectAndGiveUp)
109 MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110 if ((size_t)NewInsnID < State.MIs.size())
111 State.MIs[NewInsnID] = NewMI;
113 assert((size_t)NewInsnID == State.MIs.size() &&
114 "Expected to store MIs in order");
115 State.MIs.push_back(NewMI);
117 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
124 case GIM_CheckFeatures: {
125 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128 << ": GIM_CheckFeatures(ExpectedBitsetID="
129 << ExpectedBitsetID << ")\n");
130 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131 ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132 if (handleReject() == RejectAndGiveUp)
138 case GIM_CheckOpcode: {
139 int64_t InsnID = MatchTable[CurrentIdx++];
140 int64_t Expected = MatchTable[CurrentIdx++];
142 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143 unsigned Opcode = State.MIs[InsnID]->getOpcode();
145 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147 << "], ExpectedOpcode=" << Expected
148 << ") // Got=" << Opcode << "\n");
149 if (Opcode != Expected) {
150 if (handleReject() == RejectAndGiveUp)
156 case GIM_SwitchOpcode: {
157 int64_t InsnID = MatchTable[CurrentIdx++];
158 int64_t LowerBound = MatchTable[CurrentIdx++];
159 int64_t UpperBound = MatchTable[CurrentIdx++];
160 int64_t Default = MatchTable[CurrentIdx++];
162 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167 << LowerBound << ", " << UpperBound << "), Default=" << Default
168 << ", JumpTable...) // Got=" << Opcode << "\n";
170 if (Opcode < LowerBound || UpperBound <= Opcode) {
171 CurrentIdx = Default;
174 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176 CurrentIdx = Default;
179 OnFailResumeAt.push_back(Default);
183 case GIM_SwitchType: {
184 int64_t InsnID = MatchTable[CurrentIdx++];
185 int64_t OpIdx = MatchTable[CurrentIdx++];
186 int64_t LowerBound = MatchTable[CurrentIdx++];
187 int64_t UpperBound = MatchTable[CurrentIdx++];
188 int64_t Default = MatchTable[CurrentIdx++];
190 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196 << UpperBound << "), Default=" << Default
197 << ", JumpTable...) // Got=";
199 dbgs() << "Not a VReg\n";
201 dbgs() << MRI.getType(MO.getReg()) << "\n";
204 CurrentIdx = Default;
207 const LLT Ty = MRI.getType(MO.getReg());
208 const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209 if (TyI == ISelInfo.TypeIDMap.end()) {
210 CurrentIdx = Default;
213 const int64_t TypeID = TyI->second;
214 if (TypeID < LowerBound || UpperBound <= TypeID) {
215 CurrentIdx = Default;
218 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220 CurrentIdx = Default;
223 OnFailResumeAt.push_back(Default);
227 case GIM_CheckNumOperands: {
228 int64_t InsnID = MatchTable[CurrentIdx++];
229 int64_t Expected = MatchTable[CurrentIdx++];
230 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232 << InsnID << "], Expected=" << Expected << ")\n");
233 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234 if (State.MIs[InsnID]->getNumOperands() != Expected) {
235 if (handleReject() == RejectAndGiveUp)
240 case GIM_CheckI64ImmPredicate: {
241 int64_t InsnID = MatchTable[CurrentIdx++];
242 int64_t Predicate = MatchTable[CurrentIdx++];
243 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245 << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246 << InsnID << "], Predicate=" << Predicate << ")\n");
247 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249 "Expected G_CONSTANT");
250 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252 if (State.MIs[InsnID]->getOperand(1).isCImm())
253 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254 else if (State.MIs[InsnID]->getOperand(1).isImm())
255 Value = State.MIs[InsnID]->getOperand(1).getImm();
257 llvm_unreachable("Expected Imm or CImm operand");
259 if (!testImmPredicate_I64(Predicate, Value))
260 if (handleReject() == RejectAndGiveUp)
264 case GIM_CheckAPIntImmPredicate: {
265 int64_t InsnID = MatchTable[CurrentIdx++];
266 int64_t Predicate = MatchTable[CurrentIdx++];
267 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270 << InsnID << "], Predicate=" << Predicate << ")\n");
271 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273 "Expected G_CONSTANT");
274 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276 if (State.MIs[InsnID]->getOperand(1).isCImm())
277 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279 llvm_unreachable("Expected Imm or CImm operand");
281 if (!testImmPredicate_APInt(Predicate, Value))
282 if (handleReject() == RejectAndGiveUp)
286 case GIM_CheckAPFloatImmPredicate: {
287 int64_t InsnID = MatchTable[CurrentIdx++];
288 int64_t Predicate = MatchTable[CurrentIdx++];
289 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292 << InsnID << "], Predicate=" << Predicate << ")\n");
293 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295 "Expected G_FCONSTANT");
296 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300 if (!testImmPredicate_APFloat(Predicate, Value))
301 if (handleReject() == RejectAndGiveUp)
305 case GIM_CheckCxxInsnPredicate: {
306 int64_t InsnID = MatchTable[CurrentIdx++];
307 int64_t Predicate = MatchTable[CurrentIdx++];
308 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311 << InsnID << "], Predicate=" << Predicate << ")\n");
312 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316 if (handleReject() == RejectAndGiveUp)
320 case GIM_CheckAtomicOrdering: {
321 int64_t InsnID = MatchTable[CurrentIdx++];
322 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325 << InsnID << "], " << (uint64_t)Ordering << ")\n");
326 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327 if (!State.MIs[InsnID]->hasOneMemOperand())
328 if (handleReject() == RejectAndGiveUp)
331 for (const auto &MMO : State.MIs[InsnID]->memoperands())
332 if (MMO->getOrdering() != Ordering)
333 if (handleReject() == RejectAndGiveUp)
337 case GIM_CheckAtomicOrderingOrStrongerThan: {
338 int64_t InsnID = MatchTable[CurrentIdx++];
339 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343 << InsnID << "], " << (uint64_t)Ordering << ")\n");
344 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345 if (!State.MIs[InsnID]->hasOneMemOperand())
346 if (handleReject() == RejectAndGiveUp)
349 for (const auto &MMO : State.MIs[InsnID]->memoperands())
350 if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351 if (handleReject() == RejectAndGiveUp)
355 case GIM_CheckAtomicOrderingWeakerThan: {
356 int64_t InsnID = MatchTable[CurrentIdx++];
357 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361 << InsnID << "], " << (uint64_t)Ordering << ")\n");
362 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363 if (!State.MIs[InsnID]->hasOneMemOperand())
364 if (handleReject() == RejectAndGiveUp)
367 for (const auto &MMO : State.MIs[InsnID]->memoperands())
368 if (!isStrongerThan(Ordering, MMO->getOrdering()))
369 if (handleReject() == RejectAndGiveUp)
373 case GIM_CheckMemoryAddressSpace: {
374 int64_t InsnID = MatchTable[CurrentIdx++];
375 int64_t MMOIdx = MatchTable[CurrentIdx++];
376 // This accepts a list of possible address spaces.
377 const int NumAddrSpace = MatchTable[CurrentIdx++];
379 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380 if (handleReject() == RejectAndGiveUp)
385 // Need to still jump to the end of the list of address spaces if we find
387 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
389 const MachineMemOperand *MMO
390 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391 const unsigned MMOAddrSpace = MMO->getAddrSpace();
393 bool Success = false;
394 for (int I = 0; I != NumAddrSpace; ++I) {
395 unsigned AddrSpace = MatchTable[CurrentIdx++];
397 TgtInstructionSelector::getName(),
398 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399 << AddrSpace << '\n');
401 if (AddrSpace == MMOAddrSpace) {
407 CurrentIdx = LastIdx;
408 if (!Success && handleReject() == RejectAndGiveUp)
412 case GIM_CheckMemorySizeEqualTo: {
413 int64_t InsnID = MatchTable[CurrentIdx++];
414 int64_t MMOIdx = MatchTable[CurrentIdx++];
415 uint64_t Size = MatchTable[CurrentIdx++];
417 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
419 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420 << "]->memoperands() + " << MMOIdx
421 << ", Size=" << Size << ")\n");
422 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
424 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425 if (handleReject() == RejectAndGiveUp)
430 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
432 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433 dbgs() << MMO->getSize() << " bytes vs " << Size
435 if (MMO->getSize() != Size)
436 if (handleReject() == RejectAndGiveUp)
441 case GIM_CheckMemorySizeEqualToLLT:
442 case GIM_CheckMemorySizeLessThanLLT:
443 case GIM_CheckMemorySizeGreaterThanLLT: {
444 int64_t InsnID = MatchTable[CurrentIdx++];
445 int64_t MMOIdx = MatchTable[CurrentIdx++];
446 int64_t OpIdx = MatchTable[CurrentIdx++];
449 TgtInstructionSelector::getName(),
450 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
453 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
456 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457 << ", OpIdx=" << OpIdx << ")\n");
458 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
460 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
462 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463 dbgs() << CurrentIdx << ": Not a register\n");
464 if (handleReject() == RejectAndGiveUp)
469 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470 if (handleReject() == RejectAndGiveUp)
475 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
477 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479 MMO->getSizeInBits() != Size) {
480 if (handleReject() == RejectAndGiveUp)
482 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483 MMO->getSizeInBits() >= Size) {
484 if (handleReject() == RejectAndGiveUp)
486 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487 MMO->getSizeInBits() <= Size)
488 if (handleReject() == RejectAndGiveUp)
493 case GIM_CheckType: {
494 int64_t InsnID = MatchTable[CurrentIdx++];
495 int64_t OpIdx = MatchTable[CurrentIdx++];
496 int64_t TypeID = MatchTable[CurrentIdx++];
497 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499 << "]->getOperand(" << OpIdx
500 << "), TypeID=" << TypeID << ")\n");
501 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
504 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
505 if (handleReject() == RejectAndGiveUp)
510 case GIM_CheckPointerToAny: {
511 int64_t InsnID = MatchTable[CurrentIdx++];
512 int64_t OpIdx = MatchTable[CurrentIdx++];
513 int64_t SizeInBits = MatchTable[CurrentIdx++];
515 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517 << InsnID << "]->getOperand(" << OpIdx
518 << "), SizeInBits=" << SizeInBits << ")\n");
519 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521 const LLT Ty = MRI.getType(MO.getReg());
523 // iPTR must be looked up in the target.
524 if (SizeInBits == 0) {
525 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526 const unsigned AddrSpace = Ty.getAddressSpace();
527 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
530 assert(SizeInBits != 0 && "Pointer size must be known");
533 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534 if (handleReject() == RejectAndGiveUp)
536 } else if (handleReject() == RejectAndGiveUp)
541 case GIM_CheckRegBankForClass: {
542 int64_t InsnID = MatchTable[CurrentIdx++];
543 int64_t OpIdx = MatchTable[CurrentIdx++];
544 int64_t RCEnum = MatchTable[CurrentIdx++];
545 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547 << InsnID << "]->getOperand(" << OpIdx
548 << "), RCEnum=" << RCEnum << ")\n");
549 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
552 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554 if (handleReject() == RejectAndGiveUp)
560 case GIM_CheckComplexPattern: {
561 int64_t InsnID = MatchTable[CurrentIdx++];
562 int64_t OpIdx = MatchTable[CurrentIdx++];
563 int64_t RendererID = MatchTable[CurrentIdx++];
564 int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568 << "]->getOperand(" << OpIdx
569 << "), ComplexPredicateID=" << ComplexPredicateID
571 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572 // FIXME: Use std::invoke() when it's available.
573 ComplexRendererFns Renderer =
574 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575 State.MIs[InsnID]->getOperand(OpIdx));
576 if (Renderer.hasValue())
577 State.Renderers[RendererID] = Renderer.getValue();
579 if (handleReject() == RejectAndGiveUp)
584 case GIM_CheckConstantInt: {
585 int64_t InsnID = MatchTable[CurrentIdx++];
586 int64_t OpIdx = MatchTable[CurrentIdx++];
587 int64_t Value = MatchTable[CurrentIdx++];
588 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590 << InsnID << "]->getOperand(" << OpIdx
591 << "), Value=" << Value << ")\n");
592 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
595 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596 LLT Ty = MRI.getType(MO.getReg());
597 Value = SignExtend64(Value, Ty.getSizeInBits());
599 if (!isOperandImmEqual(MO, Value, MRI)) {
600 if (handleReject() == RejectAndGiveUp)
603 } else if (handleReject() == RejectAndGiveUp)
609 case GIM_CheckLiteralInt: {
610 int64_t InsnID = MatchTable[CurrentIdx++];
611 int64_t OpIdx = MatchTable[CurrentIdx++];
612 int64_t Value = MatchTable[CurrentIdx++];
613 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615 << InsnID << "]->getOperand(" << OpIdx
616 << "), Value=" << Value << ")\n");
617 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619 if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620 if (handleReject() == RejectAndGiveUp)
626 case GIM_CheckIntrinsicID: {
627 int64_t InsnID = MatchTable[CurrentIdx++];
628 int64_t OpIdx = MatchTable[CurrentIdx++];
629 int64_t Value = MatchTable[CurrentIdx++];
630 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632 << InsnID << "]->getOperand(" << OpIdx
633 << "), Value=" << Value << ")\n");
634 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
637 if (handleReject() == RejectAndGiveUp)
642 case GIM_CheckIsMBB: {
643 int64_t InsnID = MatchTable[CurrentIdx++];
644 int64_t OpIdx = MatchTable[CurrentIdx++];
645 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647 << "]->getOperand(" << OpIdx << "))\n");
648 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650 if (handleReject() == RejectAndGiveUp)
656 case GIM_CheckIsSafeToFold: {
657 int64_t InsnID = MatchTable[CurrentIdx++];
658 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660 << InsnID << "])\n");
661 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663 if (handleReject() == RejectAndGiveUp)
668 case GIM_CheckIsSameOperand: {
669 int64_t InsnID = MatchTable[CurrentIdx++];
670 int64_t OpIdx = MatchTable[CurrentIdx++];
671 int64_t OtherInsnID = MatchTable[CurrentIdx++];
672 int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675 << InsnID << "][" << OpIdx << "], MIs["
676 << OtherInsnID << "][" << OtherOpIdx << "])\n");
677 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679 if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680 State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681 if (handleReject() == RejectAndGiveUp)
687 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688 dbgs() << CurrentIdx << ": GIM_Reject\n");
689 if (handleReject() == RejectAndGiveUp)
693 case GIR_MutateOpcode: {
694 int64_t OldInsnID = MatchTable[CurrentIdx++];
695 uint64_t NewInsnID = MatchTable[CurrentIdx++];
696 int64_t NewOpcode = MatchTable[CurrentIdx++];
697 if (NewInsnID >= OutMIs.size())
698 OutMIs.resize(NewInsnID + 1);
700 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701 State.MIs[OldInsnID]);
702 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705 << NewInsnID << "], MIs[" << OldInsnID << "], "
706 << NewOpcode << ")\n");
711 uint64_t NewInsnID = MatchTable[CurrentIdx++];
712 int64_t Opcode = MatchTable[CurrentIdx++];
713 if (NewInsnID >= OutMIs.size())
714 OutMIs.resize(NewInsnID + 1);
716 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717 State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720 << NewInsnID << "], " << Opcode << ")\n");
725 int64_t NewInsnID = MatchTable[CurrentIdx++];
726 int64_t OldInsnID = MatchTable[CurrentIdx++];
727 int64_t OpIdx = MatchTable[CurrentIdx++];
728 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
732 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
737 case GIR_CopyOrAddZeroReg: {
738 int64_t NewInsnID = MatchTable[CurrentIdx++];
739 int64_t OldInsnID = MatchTable[CurrentIdx++];
740 int64_t OpIdx = MatchTable[CurrentIdx++];
741 int64_t ZeroReg = MatchTable[CurrentIdx++];
742 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744 if (isOperandImmEqual(MO, 0, MRI))
745 OutMIs[NewInsnID].addReg(ZeroReg);
747 OutMIs[NewInsnID].add(MO);
748 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750 << NewInsnID << "], MIs[" << OldInsnID << "], "
751 << OpIdx << ", " << ZeroReg << ")\n");
755 case GIR_CopySubReg: {
756 int64_t NewInsnID = MatchTable[CurrentIdx++];
757 int64_t OldInsnID = MatchTable[CurrentIdx++];
758 int64_t OpIdx = MatchTable[CurrentIdx++];
759 int64_t SubRegIdx = MatchTable[CurrentIdx++];
760 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
763 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765 << NewInsnID << "], MIs[" << OldInsnID << "], "
766 << OpIdx << ", " << SubRegIdx << ")\n");
770 case GIR_AddImplicitDef: {
771 int64_t InsnID = MatchTable[CurrentIdx++];
772 int64_t RegNum = MatchTable[CurrentIdx++];
773 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774 OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777 << InsnID << "], " << RegNum << ")\n");
781 case GIR_AddImplicitUse: {
782 int64_t InsnID = MatchTable[CurrentIdx++];
783 int64_t RegNum = MatchTable[CurrentIdx++];
784 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788 << InsnID << "], " << RegNum << ")\n");
792 case GIR_AddRegister: {
793 int64_t InsnID = MatchTable[CurrentIdx++];
794 int64_t RegNum = MatchTable[CurrentIdx++];
795 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796 OutMIs[InsnID].addReg(RegNum);
797 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799 << InsnID << "], " << RegNum << ")\n");
803 case GIR_AddTempRegister: {
804 int64_t InsnID = MatchTable[CurrentIdx++];
805 int64_t TempRegID = MatchTable[CurrentIdx++];
806 uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811 << InsnID << "], TempRegisters[" << TempRegID
812 << "], " << TempRegFlags << ")\n");
817 int64_t InsnID = MatchTable[CurrentIdx++];
818 int64_t Imm = MatchTable[CurrentIdx++];
819 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820 OutMIs[InsnID].addImm(Imm);
821 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823 << "], " << Imm << ")\n");
827 case GIR_ComplexRenderer: {
828 int64_t InsnID = MatchTable[CurrentIdx++];
829 int64_t RendererID = MatchTable[CurrentIdx++];
830 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831 for (const auto &RenderOpFn : State.Renderers[RendererID])
832 RenderOpFn(OutMIs[InsnID]);
833 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835 << InsnID << "], " << RendererID << ")\n");
838 case GIR_ComplexSubOperandRenderer: {
839 int64_t InsnID = MatchTable[CurrentIdx++];
840 int64_t RendererID = MatchTable[CurrentIdx++];
841 int64_t RenderOpID = MatchTable[CurrentIdx++];
842 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
846 << ": GIR_ComplexSubOperandRenderer(OutMIs["
847 << InsnID << "], " << RendererID << ", "
848 << RenderOpID << ")\n");
852 case GIR_CopyConstantAsSImm: {
853 int64_t NewInsnID = MatchTable[CurrentIdx++];
854 int64_t OldInsnID = MatchTable[CurrentIdx++];
855 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858 OutMIs[NewInsnID].addImm(
859 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
861 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
863 llvm_unreachable("Expected Imm or CImm operand");
864 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
870 // TODO: Needs a test case once we have a pattern that uses this.
871 case GIR_CopyFConstantAsFPImm: {
872 int64_t NewInsnID = MatchTable[CurrentIdx++];
873 int64_t OldInsnID = MatchTable[CurrentIdx++];
874 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877 OutMIs[NewInsnID].addFPImm(
878 State.MIs[OldInsnID]->getOperand(1).getFPImm());
880 llvm_unreachable("Expected FPImm operand");
881 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
887 case GIR_CustomRenderer: {
888 int64_t InsnID = MatchTable[CurrentIdx++];
889 int64_t OldInsnID = MatchTable[CurrentIdx++];
890 int64_t RendererFnID = MatchTable[CurrentIdx++];
891 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894 << InsnID << "], MIs[" << OldInsnID << "], "
895 << RendererFnID << ")\n");
896 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897 *State.MIs[OldInsnID]);
900 case GIR_ConstrainOperandRC: {
901 int64_t InsnID = MatchTable[CurrentIdx++];
902 int64_t OpIdx = MatchTable[CurrentIdx++];
903 int64_t RCEnum = MatchTable[CurrentIdx++];
904 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905 constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906 *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909 << InsnID << "], " << OpIdx << ", " << RCEnum
914 case GIR_ConstrainSelectedInstOperands: {
915 int64_t InsnID = MatchTable[CurrentIdx++];
916 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
919 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
921 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922 << InsnID << "])\n");
926 case GIR_MergeMemOperands: {
927 int64_t InsnID = MatchTable[CurrentIdx++];
928 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
930 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
933 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934 while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935 GIU_MergeMemOperands_EndOfList) {
936 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937 dbgs() << ", MIs[" << MergeInsnID << "]");
938 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939 OutMIs[InsnID].addMemOperand(MMO);
941 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
945 case GIR_EraseFromParent: {
946 int64_t InsnID = MatchTable[CurrentIdx++];
947 assert(State.MIs[InsnID] &&
948 "Attempted to erase an undefined instruction");
949 State.MIs[InsnID]->eraseFromParent();
950 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952 << InsnID << "])\n");
956 case GIR_MakeTempReg: {
957 int64_t TempRegID = MatchTable[CurrentIdx++];
958 int64_t TypeID = MatchTable[CurrentIdx++];
960 State.TempRegisters[TempRegID] =
961 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
969 int64_t RuleID = MatchTable[CurrentIdx++];
970 CoverageInfo.setCovered(RuleID);
972 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
974 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
979 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980 dbgs() << CurrentIdx << ": GIR_Done\n");
984 llvm_unreachable("Unexpected command");
989 } // end namespace llvm
991 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H