1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// \file This file implements the LegalizerHelper class to legalize
11 /// individual instructions and the LegalizeMachineIR wrapper pass for the
12 /// primary legalization.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
17 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/CodeGen/TargetSubtargetInfo.h"
22 #include "llvm/Support/Debug.h"
23 #include "llvm/Support/raw_ostream.h"
26 #define DEBUG_TYPE "legalizer"
30 LegalizerHelper::LegalizerHelper(MachineFunction &MF)
31 : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
35 LegalizerHelper::LegalizeResult
36 LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
37 DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
39 auto Action = LI.getAction(MI, MRI);
40 switch (std::get<0>(Action)) {
41 case LegalizerInfo::Legal:
42 DEBUG(dbgs() << ".. Already legal\n");
44 case LegalizerInfo::Libcall:
45 DEBUG(dbgs() << ".. Convert to libcall\n");
47 case LegalizerInfo::NarrowScalar:
48 DEBUG(dbgs() << ".. Narrow scalar\n");
49 return narrowScalar(MI, std::get<1>(Action), std::get<2>(Action));
50 case LegalizerInfo::WidenScalar:
51 DEBUG(dbgs() << ".. Widen scalar\n");
52 return widenScalar(MI, std::get<1>(Action), std::get<2>(Action));
53 case LegalizerInfo::Lower:
54 DEBUG(dbgs() << ".. Lower\n");
55 return lower(MI, std::get<1>(Action), std::get<2>(Action));
56 case LegalizerInfo::FewerElements:
57 DEBUG(dbgs() << ".. Reduce number of elements\n");
58 return fewerElementsVector(MI, std::get<1>(Action), std::get<2>(Action));
59 case LegalizerInfo::Custom:
60 DEBUG(dbgs() << ".. Custom legalization\n");
61 return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
64 DEBUG(dbgs() << ".. Unable to legalize\n");
65 return UnableToLegalize;
69 void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
70 SmallVectorImpl<unsigned> &VRegs) {
71 for (int i = 0; i < NumParts; ++i)
72 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
73 MIRBuilder.buildUnmerge(VRegs, Reg);
76 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
78 case TargetOpcode::G_SDIV:
79 assert(Size == 32 && "Unsupported size");
80 return RTLIB::SDIV_I32;
81 case TargetOpcode::G_UDIV:
82 assert(Size == 32 && "Unsupported size");
83 return RTLIB::UDIV_I32;
84 case TargetOpcode::G_SREM:
85 assert(Size == 32 && "Unsupported size");
86 return RTLIB::SREM_I32;
87 case TargetOpcode::G_UREM:
88 assert(Size == 32 && "Unsupported size");
89 return RTLIB::UREM_I32;
90 case TargetOpcode::G_FADD:
91 assert((Size == 32 || Size == 64) && "Unsupported size");
92 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
93 case TargetOpcode::G_FSUB:
94 assert((Size == 32 || Size == 64) && "Unsupported size");
95 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
96 case TargetOpcode::G_FMUL:
97 assert((Size == 32 || Size == 64) && "Unsupported size");
98 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
99 case TargetOpcode::G_FDIV:
100 assert((Size == 32 || Size == 64) && "Unsupported size");
101 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
102 case TargetOpcode::G_FREM:
103 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
104 case TargetOpcode::G_FPOW:
105 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
107 llvm_unreachable("Unknown libcall function");
110 LegalizerHelper::LegalizeResult
111 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
112 const CallLowering::ArgInfo &Result,
113 ArrayRef<CallLowering::ArgInfo> Args) {
114 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
115 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
116 const char *Name = TLI.getLibcallName(Libcall);
118 MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
119 if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
120 MachineOperand::CreateES(Name), Result, Args))
121 return LegalizerHelper::UnableToLegalize;
123 return LegalizerHelper::Legalized;
126 static LegalizerHelper::LegalizeResult
127 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
129 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
130 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
131 {{MI.getOperand(1).getReg(), OpType},
132 {MI.getOperand(2).getReg(), OpType}});
135 LegalizerHelper::LegalizeResult
136 LegalizerHelper::libcall(MachineInstr &MI) {
137 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
138 unsigned Size = LLTy.getSizeInBits();
139 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
141 MIRBuilder.setInstr(MI);
143 switch (MI.getOpcode()) {
145 return UnableToLegalize;
146 case TargetOpcode::G_SDIV:
147 case TargetOpcode::G_UDIV:
148 case TargetOpcode::G_SREM:
149 case TargetOpcode::G_UREM: {
150 Type *HLTy = Type::getInt32Ty(Ctx);
151 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
152 if (Status != Legalized)
156 case TargetOpcode::G_FADD:
157 case TargetOpcode::G_FSUB:
158 case TargetOpcode::G_FMUL:
159 case TargetOpcode::G_FDIV:
160 case TargetOpcode::G_FPOW:
161 case TargetOpcode::G_FREM: {
162 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
163 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
164 if (Status != Legalized)
170 MI.eraseFromParent();
174 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
177 // FIXME: Don't know how to handle secondary types yet.
178 if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
179 return UnableToLegalize;
181 MIRBuilder.setInstr(MI);
183 int64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
184 int64_t NarrowSize = NarrowTy.getSizeInBits();
186 switch (MI.getOpcode()) {
188 return UnableToLegalize;
189 case TargetOpcode::G_IMPLICIT_DEF: {
190 // FIXME: add support for when SizeOp0 isn't an exact multiple of
192 if (SizeOp0 % NarrowSize != 0)
193 return UnableToLegalize;
194 int NumParts = SizeOp0 / NarrowSize;
196 SmallVector<unsigned, 2> DstRegs;
197 for (int i = 0; i < NumParts; ++i) {
198 unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy);
199 MIRBuilder.buildUndef(Dst);
200 DstRegs.push_back(Dst);
202 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
203 MI.eraseFromParent();
206 case TargetOpcode::G_ADD: {
207 // FIXME: add support for when SizeOp0 isn't an exact multiple of
209 if (SizeOp0 % NarrowSize != 0)
210 return UnableToLegalize;
211 // Expand in terms of carry-setting/consuming G_ADDE instructions.
212 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
214 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
215 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
216 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
218 unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
219 MIRBuilder.buildConstant(CarryIn, 0);
221 for (int i = 0; i < NumParts; ++i) {
222 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
223 unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
225 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
226 Src2Regs[i], CarryIn);
228 DstRegs.push_back(DstReg);
231 unsigned DstReg = MI.getOperand(0).getReg();
232 MIRBuilder.buildMerge(DstReg, DstRegs);
233 MI.eraseFromParent();
236 case TargetOpcode::G_EXTRACT: {
238 return UnableToLegalize;
240 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
241 // FIXME: add support for when SizeOp1 isn't an exact multiple of
243 if (SizeOp1 % NarrowSize != 0)
244 return UnableToLegalize;
245 int NumParts = SizeOp1 / NarrowSize;
247 SmallVector<unsigned, 2> SrcRegs, DstRegs;
248 SmallVector<uint64_t, 2> Indexes;
249 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
251 unsigned OpReg = MI.getOperand(0).getReg();
252 int64_t OpStart = MI.getOperand(2).getImm();
253 int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
254 for (int i = 0; i < NumParts; ++i) {
255 unsigned SrcStart = i * NarrowSize;
257 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
258 // No part of the extract uses this subregister, ignore it.
260 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
261 // The entire subregister is extracted, forward the value.
262 DstRegs.push_back(SrcRegs[i]);
266 // OpSegStart is where this destination segment would start in OpReg if it
267 // extended infinitely in both directions.
268 int64_t ExtractOffset, SegSize;
269 if (OpStart < SrcStart) {
271 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
273 ExtractOffset = OpStart - SrcStart;
274 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
277 unsigned SegReg = SrcRegs[i];
278 if (ExtractOffset != 0 || SegSize != NarrowSize) {
279 // A genuine extract is needed.
280 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
281 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
284 DstRegs.push_back(SegReg);
287 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
288 MI.eraseFromParent();
291 case TargetOpcode::G_INSERT: {
292 // FIXME: add support for when SizeOp0 isn't an exact multiple of
294 if (SizeOp0 % NarrowSize != 0)
295 return UnableToLegalize;
297 int NumParts = SizeOp0 / NarrowSize;
299 SmallVector<unsigned, 2> SrcRegs, DstRegs;
300 SmallVector<uint64_t, 2> Indexes;
301 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
303 unsigned OpReg = MI.getOperand(2).getReg();
304 int64_t OpStart = MI.getOperand(3).getImm();
305 int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
306 for (int i = 0; i < NumParts; ++i) {
307 unsigned DstStart = i * NarrowSize;
309 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
310 // No part of the insert affects this subregister, forward the original.
311 DstRegs.push_back(SrcRegs[i]);
313 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
314 // The entire subregister is defined by this insert, forward the new
316 DstRegs.push_back(OpReg);
320 // OpSegStart is where this destination segment would start in OpReg if it
321 // extended infinitely in both directions.
322 int64_t ExtractOffset, InsertOffset, SegSize;
323 if (OpStart < DstStart) {
325 ExtractOffset = DstStart - OpStart;
326 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
328 InsertOffset = OpStart - DstStart;
331 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
334 unsigned SegReg = OpReg;
335 if (ExtractOffset != 0 || SegSize != OpSize) {
336 // A genuine extract is needed.
337 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
338 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
341 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
342 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
343 DstRegs.push_back(DstReg);
346 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
347 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
348 MI.eraseFromParent();
351 case TargetOpcode::G_LOAD: {
352 // FIXME: add support for when SizeOp0 isn't an exact multiple of
354 if (SizeOp0 % NarrowSize != 0)
355 return UnableToLegalize;
356 int NumParts = SizeOp0 / NarrowSize;
357 LLT OffsetTy = LLT::scalar(
358 MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
360 SmallVector<unsigned, 2> DstRegs;
361 for (int i = 0; i < NumParts; ++i) {
362 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
364 unsigned Adjustment = i * NarrowSize / 8;
366 MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
369 // TODO: This is conservatively correct, but we probably want to split the
370 // memory operands in the future.
371 MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
373 DstRegs.push_back(DstReg);
375 unsigned DstReg = MI.getOperand(0).getReg();
376 MIRBuilder.buildMerge(DstReg, DstRegs);
377 MI.eraseFromParent();
380 case TargetOpcode::G_STORE: {
381 // FIXME: add support for when SizeOp0 isn't an exact multiple of
383 if (SizeOp0 % NarrowSize != 0)
384 return UnableToLegalize;
385 int NumParts = SizeOp0 / NarrowSize;
386 LLT OffsetTy = LLT::scalar(
387 MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
389 SmallVector<unsigned, 2> SrcRegs;
390 extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
392 for (int i = 0; i < NumParts; ++i) {
394 unsigned Adjustment = i * NarrowSize / 8;
396 MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
399 // TODO: This is conservatively correct, but we probably want to split the
400 // memory operands in the future.
401 MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
403 MI.eraseFromParent();
406 case TargetOpcode::G_CONSTANT: {
407 // FIXME: add support for when SizeOp0 isn't an exact multiple of
409 if (SizeOp0 % NarrowSize != 0)
410 return UnableToLegalize;
411 int NumParts = SizeOp0 / NarrowSize;
412 const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
413 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
415 SmallVector<unsigned, 2> DstRegs;
416 for (int i = 0; i < NumParts; ++i) {
417 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
419 ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
420 MIRBuilder.buildConstant(DstReg, *CI);
421 DstRegs.push_back(DstReg);
423 unsigned DstReg = MI.getOperand(0).getReg();
424 MIRBuilder.buildMerge(DstReg, DstRegs);
425 MI.eraseFromParent();
428 case TargetOpcode::G_OR: {
429 // Legalize bitwise operation:
430 // A = BinOp<Ty> B, C
432 // B1, ..., BN = G_UNMERGE_VALUES B
433 // C1, ..., CN = G_UNMERGE_VALUES C
434 // A1 = BinOp<Ty/N> B1, C2
436 // AN = BinOp<Ty/N> BN, CN
437 // A = G_MERGE_VALUES A1, ..., AN
439 // FIXME: add support for when SizeOp0 isn't an exact multiple of
441 if (SizeOp0 % NarrowSize != 0)
442 return UnableToLegalize;
443 int NumParts = SizeOp0 / NarrowSize;
445 // List the registers where the destination will be scattered.
446 SmallVector<unsigned, 2> DstRegs;
447 // List the registers where the first argument will be split.
448 SmallVector<unsigned, 2> SrcsReg1;
449 // List the registers where the second argument will be split.
450 SmallVector<unsigned, 2> SrcsReg2;
451 // Create all the temporary registers.
452 for (int i = 0; i < NumParts; ++i) {
453 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
454 unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy);
455 unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy);
457 DstRegs.push_back(DstReg);
458 SrcsReg1.push_back(SrcReg1);
459 SrcsReg2.push_back(SrcReg2);
461 // Explode the big arguments into smaller chunks.
462 MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg());
463 MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg());
465 // Do the operation on each small part.
466 for (int i = 0; i < NumParts; ++i)
467 MIRBuilder.buildOr(DstRegs[i], SrcsReg1[i], SrcsReg2[i]);
469 // Gather the destination registers into the final destination.
470 unsigned DstReg = MI.getOperand(0).getReg();
471 MIRBuilder.buildMerge(DstReg, DstRegs);
472 MI.eraseFromParent();
478 LegalizerHelper::LegalizeResult
479 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
480 MIRBuilder.setInstr(MI);
482 switch (MI.getOpcode()) {
484 return UnableToLegalize;
485 case TargetOpcode::G_ADD:
486 case TargetOpcode::G_AND:
487 case TargetOpcode::G_MUL:
488 case TargetOpcode::G_OR:
489 case TargetOpcode::G_XOR:
490 case TargetOpcode::G_SUB:
491 case TargetOpcode::G_SHL: {
492 // Perform operation at larger width (any extension is fine here, high bits
493 // don't affect the result) and then truncate the result back to the
495 unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
496 unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
497 MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(1).getReg());
498 MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(2).getReg());
500 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
501 MIRBuilder.buildInstr(MI.getOpcode())
506 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
507 MI.eraseFromParent();
510 case TargetOpcode::G_SDIV:
511 case TargetOpcode::G_UDIV:
512 case TargetOpcode::G_SREM:
513 case TargetOpcode::G_UREM:
514 case TargetOpcode::G_ASHR:
515 case TargetOpcode::G_LSHR: {
516 unsigned ExtOp = MI.getOpcode() == TargetOpcode::G_SDIV ||
517 MI.getOpcode() == TargetOpcode::G_SREM ||
518 MI.getOpcode() == TargetOpcode::G_ASHR
519 ? TargetOpcode::G_SEXT
520 : TargetOpcode::G_ZEXT;
522 unsigned LHSExt = MRI.createGenericVirtualRegister(WideTy);
523 MIRBuilder.buildInstr(ExtOp).addDef(LHSExt).addUse(
524 MI.getOperand(1).getReg());
526 unsigned RHSExt = MRI.createGenericVirtualRegister(WideTy);
527 MIRBuilder.buildInstr(ExtOp).addDef(RHSExt).addUse(
528 MI.getOperand(2).getReg());
530 unsigned ResExt = MRI.createGenericVirtualRegister(WideTy);
531 MIRBuilder.buildInstr(MI.getOpcode())
536 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), ResExt);
537 MI.eraseFromParent();
540 case TargetOpcode::G_SELECT: {
542 return UnableToLegalize;
544 // Perform operation at larger width (any extension is fine here, high bits
545 // don't affect the result) and then truncate the result back to the
547 unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
548 unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
549 MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(2).getReg());
550 MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(3).getReg());
552 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
553 MIRBuilder.buildInstr(TargetOpcode::G_SELECT)
555 .addReg(MI.getOperand(1).getReg())
559 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
560 MI.eraseFromParent();
563 case TargetOpcode::G_FPTOSI:
564 case TargetOpcode::G_FPTOUI: {
566 return UnableToLegalize;
568 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
569 MIRBuilder.buildInstr(MI.getOpcode())
571 .addUse(MI.getOperand(1).getReg());
573 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
574 MI.eraseFromParent();
577 case TargetOpcode::G_SITOFP:
578 case TargetOpcode::G_UITOFP: {
580 return UnableToLegalize;
582 unsigned Src = MI.getOperand(1).getReg();
583 unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
585 if (MI.getOpcode() == TargetOpcode::G_SITOFP) {
586 MIRBuilder.buildSExt(SrcExt, Src);
588 assert(MI.getOpcode() == TargetOpcode::G_UITOFP && "Unexpected conv op");
589 MIRBuilder.buildZExt(SrcExt, Src);
592 MIRBuilder.buildInstr(MI.getOpcode())
593 .addDef(MI.getOperand(0).getReg())
596 MI.eraseFromParent();
599 case TargetOpcode::G_INSERT: {
601 return UnableToLegalize;
603 unsigned Src = MI.getOperand(1).getReg();
604 unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
605 MIRBuilder.buildAnyExt(SrcExt, Src);
607 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
608 auto MIB = MIRBuilder.buildInsert(DstExt, SrcExt, MI.getOperand(2).getReg(),
609 MI.getOperand(3).getImm());
610 for (unsigned OpNum = 4; OpNum < MI.getNumOperands(); OpNum += 2) {
611 MIB.addReg(MI.getOperand(OpNum).getReg());
612 MIB.addImm(MI.getOperand(OpNum + 1).getImm());
615 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
616 MI.eraseFromParent();
619 case TargetOpcode::G_LOAD: {
620 assert(alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) ==
621 WideTy.getSizeInBits() &&
622 "illegal to increase number of bytes loaded");
624 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
625 MIRBuilder.buildLoad(DstExt, MI.getOperand(1).getReg(),
626 **MI.memoperands_begin());
627 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
628 MI.eraseFromParent();
631 case TargetOpcode::G_STORE: {
632 if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
633 WideTy != LLT::scalar(8))
634 return UnableToLegalize;
636 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
637 auto Content = TLI.getBooleanContents(false, false);
639 unsigned ExtOp = TargetOpcode::G_ANYEXT;
640 if (Content == TargetLoweringBase::ZeroOrOneBooleanContent)
641 ExtOp = TargetOpcode::G_ZEXT;
642 else if (Content == TargetLoweringBase::ZeroOrNegativeOneBooleanContent)
643 ExtOp = TargetOpcode::G_SEXT;
645 ExtOp = TargetOpcode::G_ANYEXT;
647 unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
648 MIRBuilder.buildInstr(ExtOp).addDef(SrcExt).addUse(
649 MI.getOperand(0).getReg());
650 MIRBuilder.buildStore(SrcExt, MI.getOperand(1).getReg(),
651 **MI.memoperands_begin());
652 MI.eraseFromParent();
655 case TargetOpcode::G_CONSTANT: {
656 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
657 MIRBuilder.buildConstant(DstExt, *MI.getOperand(1).getCImm());
658 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
659 MI.eraseFromParent();
662 case TargetOpcode::G_FCONSTANT: {
663 unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
664 const ConstantFP *CFP = MI.getOperand(1).getFPImm();
665 APFloat Val = CFP->getValueAPF();
666 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
667 auto LLT2Sem = [](LLT Ty) {
668 switch (Ty.getSizeInBits()) {
670 return &APFloat::IEEEsingle();
673 return &APFloat::IEEEdouble();
676 llvm_unreachable("Unhandled fp widen type");
680 Val.convert(*LLT2Sem(WideTy), APFloat::rmTowardZero, &LosesInfo);
681 MIRBuilder.buildFConstant(DstExt, *ConstantFP::get(Ctx, Val));
682 MIRBuilder.buildFPTrunc(MI.getOperand(0).getReg(), DstExt);
683 MI.eraseFromParent();
686 case TargetOpcode::G_BRCOND: {
687 unsigned TstExt = MRI.createGenericVirtualRegister(WideTy);
688 MIRBuilder.buildAnyExt(TstExt, MI.getOperand(0).getReg());
689 MIRBuilder.buildBrCond(TstExt, *MI.getOperand(1).getMBB());
690 MI.eraseFromParent();
693 case TargetOpcode::G_FCMP: {
694 unsigned Op0Ext, Op1Ext, DstReg;
695 unsigned Cmp1 = MI.getOperand(2).getReg();
696 unsigned Cmp2 = MI.getOperand(3).getReg();
700 DstReg = MRI.createGenericVirtualRegister(WideTy);
702 Op0Ext = MRI.createGenericVirtualRegister(WideTy);
703 Op1Ext = MRI.createGenericVirtualRegister(WideTy);
704 DstReg = MI.getOperand(0).getReg();
705 MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op0Ext, Cmp1);
706 MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op1Ext, Cmp2);
708 MIRBuilder.buildFCmp(
709 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
710 DstReg, Op0Ext, Op1Ext);
712 MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
714 MI.eraseFromParent();
717 case TargetOpcode::G_ICMP: {
718 bool IsSigned = CmpInst::isSigned(
719 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()));
720 unsigned Cmp1 = MI.getOperand(2).getReg();
721 unsigned Cmp2 = MI.getOperand(3).getReg();
722 unsigned Op0Ext, Op1Ext, DstReg;
726 DstReg = MRI.createGenericVirtualRegister(WideTy);
728 Op0Ext = MRI.createGenericVirtualRegister(WideTy);
729 Op1Ext = MRI.createGenericVirtualRegister(WideTy);
730 DstReg = MI.getOperand(0).getReg();
732 MIRBuilder.buildSExt(Op0Ext, Cmp1);
733 MIRBuilder.buildSExt(Op1Ext, Cmp2);
735 MIRBuilder.buildZExt(Op0Ext, Cmp1);
736 MIRBuilder.buildZExt(Op1Ext, Cmp2);
739 MIRBuilder.buildICmp(
740 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
741 DstReg, Op0Ext, Op1Ext);
743 MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
745 MI.eraseFromParent();
748 case TargetOpcode::G_GEP: {
749 assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
750 unsigned OffsetExt = MRI.createGenericVirtualRegister(WideTy);
751 MIRBuilder.buildSExt(OffsetExt, MI.getOperand(2).getReg());
752 MI.getOperand(2).setReg(OffsetExt);
755 case TargetOpcode::G_PHI: {
756 assert(TypeIdx == 0 && "Expecting only Idx 0");
757 auto getExtendedReg = [&](unsigned Reg, MachineBasicBlock &MBB) {
758 auto FirstTermIt = MBB.getFirstTerminator();
759 MIRBuilder.setInsertPt(MBB, FirstTermIt);
760 MachineInstr *DefMI = MRI.getVRegDef(Reg);
761 MachineInstrBuilder MIB;
762 if (DefMI->getOpcode() == TargetOpcode::G_TRUNC)
763 MIB = MIRBuilder.buildAnyExtOrTrunc(WideTy,
764 DefMI->getOperand(1).getReg());
766 MIB = MIRBuilder.buildAnyExt(WideTy, Reg);
767 return MIB->getOperand(0).getReg();
769 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, WideTy);
770 for (auto OpIt = MI.operands_begin() + 1, OpE = MI.operands_end();
772 unsigned Reg = OpIt++->getReg();
773 MachineBasicBlock *OpMBB = OpIt++->getMBB();
774 MIB.addReg(getExtendedReg(Reg, *OpMBB));
777 auto *MBB = MI.getParent();
778 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
779 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(),
780 MIB->getOperand(0).getReg());
781 MI.eraseFromParent();
787 LegalizerHelper::LegalizeResult
788 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
789 using namespace TargetOpcode;
790 MIRBuilder.setInstr(MI);
792 switch(MI.getOpcode()) {
794 return UnableToLegalize;
795 case TargetOpcode::G_SREM:
796 case TargetOpcode::G_UREM: {
797 unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
798 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
800 .addUse(MI.getOperand(1).getReg())
801 .addUse(MI.getOperand(2).getReg());
803 unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
804 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
805 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
807 MI.eraseFromParent();
810 case TargetOpcode::G_SMULO:
811 case TargetOpcode::G_UMULO: {
812 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
814 unsigned Res = MI.getOperand(0).getReg();
815 unsigned Overflow = MI.getOperand(1).getReg();
816 unsigned LHS = MI.getOperand(2).getReg();
817 unsigned RHS = MI.getOperand(3).getReg();
819 MIRBuilder.buildMul(Res, LHS, RHS);
821 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
822 ? TargetOpcode::G_SMULH
823 : TargetOpcode::G_UMULH;
825 unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
826 MIRBuilder.buildInstr(Opcode)
831 unsigned Zero = MRI.createGenericVirtualRegister(Ty);
832 MIRBuilder.buildConstant(Zero, 0);
834 // For *signed* multiply, overflow is detected by checking:
835 // (hi != (lo >> bitwidth-1))
836 if (Opcode == TargetOpcode::G_SMULH) {
837 unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
838 unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
839 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
840 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
844 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
846 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
848 MI.eraseFromParent();
851 case TargetOpcode::G_FNEG: {
852 // TODO: Handle vector types once we are able to
855 return UnableToLegalize;
856 unsigned Res = MI.getOperand(0).getReg();
858 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
859 switch (Ty.getSizeInBits()) {
861 ZeroTy = Type::getHalfTy(Ctx);
864 ZeroTy = Type::getFloatTy(Ctx);
867 ZeroTy = Type::getDoubleTy(Ctx);
870 ZeroTy = Type::getFP128Ty(Ctx);
873 llvm_unreachable("unexpected floating-point type");
875 ConstantFP &ZeroForNegation =
876 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
877 unsigned Zero = MRI.createGenericVirtualRegister(Ty);
878 MIRBuilder.buildFConstant(Zero, ZeroForNegation);
879 MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
882 .addUse(MI.getOperand(1).getReg());
883 MI.eraseFromParent();
886 case TargetOpcode::G_FSUB: {
887 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
888 // First, check if G_FNEG is marked as Lower. If so, we may
889 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
890 if (LI.getAction({G_FNEG, Ty}).first == LegalizerInfo::Lower)
891 return UnableToLegalize;
892 unsigned Res = MI.getOperand(0).getReg();
893 unsigned LHS = MI.getOperand(1).getReg();
894 unsigned RHS = MI.getOperand(2).getReg();
895 unsigned Neg = MRI.createGenericVirtualRegister(Ty);
896 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
897 MIRBuilder.buildInstr(TargetOpcode::G_FADD)
901 MI.eraseFromParent();
904 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
905 unsigned OldValRes = MI.getOperand(0).getReg();
906 unsigned SuccessRes = MI.getOperand(1).getReg();
907 unsigned Addr = MI.getOperand(2).getReg();
908 unsigned CmpVal = MI.getOperand(3).getReg();
909 unsigned NewVal = MI.getOperand(4).getReg();
910 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
911 **MI.memoperands_begin());
912 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
913 MI.eraseFromParent();
919 LegalizerHelper::LegalizeResult
920 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
922 // FIXME: Don't know how to handle secondary types yet.
924 return UnableToLegalize;
925 switch (MI.getOpcode()) {
927 return UnableToLegalize;
928 case TargetOpcode::G_ADD: {
929 unsigned NarrowSize = NarrowTy.getSizeInBits();
930 unsigned DstReg = MI.getOperand(0).getReg();
931 unsigned Size = MRI.getType(DstReg).getSizeInBits();
932 int NumParts = Size / NarrowSize;
933 // FIXME: Don't know how to handle the situation where the small vectors
934 // aren't all the same size yet.
935 if (Size % NarrowSize != 0)
936 return UnableToLegalize;
938 MIRBuilder.setInstr(MI);
940 SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
941 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
942 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
944 for (int i = 0; i < NumParts; ++i) {
945 unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
946 MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
947 DstRegs.push_back(DstReg);
950 MIRBuilder.buildMerge(DstReg, DstRegs);
951 MI.eraseFromParent();