1 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// This file implements the IRTranslator class.
11 //===----------------------------------------------------------------------===//
13 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/CodeGen/Analysis.h"
20 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
21 #include "llvm/CodeGen/LowLevelType.h"
22 #include "llvm/CodeGen/MachineBasicBlock.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineOperand.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/StackProtector.h"
30 #include "llvm/CodeGen/TargetFrameLowering.h"
31 #include "llvm/CodeGen/TargetLowering.h"
32 #include "llvm/CodeGen/TargetPassConfig.h"
33 #include "llvm/CodeGen/TargetRegisterInfo.h"
34 #include "llvm/CodeGen/TargetSubtargetInfo.h"
35 #include "llvm/IR/BasicBlock.h"
36 #include "llvm/IR/Constant.h"
37 #include "llvm/IR/Constants.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/IR/DebugInfo.h"
40 #include "llvm/IR/DerivedTypes.h"
41 #include "llvm/IR/Function.h"
42 #include "llvm/IR/GetElementPtrTypeIterator.h"
43 #include "llvm/IR/InlineAsm.h"
44 #include "llvm/IR/InstrTypes.h"
45 #include "llvm/IR/Instructions.h"
46 #include "llvm/IR/IntrinsicInst.h"
47 #include "llvm/IR/Intrinsics.h"
48 #include "llvm/IR/LLVMContext.h"
49 #include "llvm/IR/Metadata.h"
50 #include "llvm/IR/Type.h"
51 #include "llvm/IR/User.h"
52 #include "llvm/IR/Value.h"
53 #include "llvm/MC/MCContext.h"
54 #include "llvm/Pass.h"
55 #include "llvm/Support/Casting.h"
56 #include "llvm/Support/CodeGen.h"
57 #include "llvm/Support/Debug.h"
58 #include "llvm/Support/ErrorHandling.h"
59 #include "llvm/Support/LowLevelTypeImpl.h"
60 #include "llvm/Support/MathExtras.h"
61 #include "llvm/Support/raw_ostream.h"
62 #include "llvm/Target/TargetIntrinsicInfo.h"
63 #include "llvm/Target/TargetMachine.h"
72 #define DEBUG_TYPE "irtranslator"
76 char IRTranslator::ID = 0;
78 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
80 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
81 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
84 static void reportTranslationError(MachineFunction &MF,
85 const TargetPassConfig &TPC,
86 OptimizationRemarkEmitter &ORE,
87 OptimizationRemarkMissed &R) {
88 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
90 // Print the function name explicitly if we don't have a debug location (which
91 // makes the diagnostic less useful) or if we're going to emit a raw error.
92 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
93 R << (" (in function: " + MF.getName() + ")").str();
95 if (TPC.isGlobalISelAbortEnabled())
96 report_fatal_error(R.getMsg());
101 IRTranslator::IRTranslator() : MachineFunctionPass(ID) {
102 initializeIRTranslatorPass(*PassRegistry::getPassRegistry());
105 void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
106 AU.addRequired<StackProtector>();
107 AU.addRequired<TargetPassConfig>();
108 getSelectionDAGFallbackAnalysisUsage(AU);
109 MachineFunctionPass::getAnalysisUsage(AU);
112 static void computeValueLLTs(const DataLayout &DL, Type &Ty,
113 SmallVectorImpl<LLT> &ValueTys,
114 SmallVectorImpl<uint64_t> *Offsets = nullptr,
115 uint64_t StartingOffset = 0) {
116 // Given a struct type, recursively traverse the elements.
117 if (StructType *STy = dyn_cast<StructType>(&Ty)) {
118 const StructLayout *SL = DL.getStructLayout(STy);
119 for (unsigned I = 0, E = STy->getNumElements(); I != E; ++I)
120 computeValueLLTs(DL, *STy->getElementType(I), ValueTys, Offsets,
121 StartingOffset + SL->getElementOffset(I));
124 // Given an array type, recursively traverse the elements.
125 if (ArrayType *ATy = dyn_cast<ArrayType>(&Ty)) {
126 Type *EltTy = ATy->getElementType();
127 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
128 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
129 computeValueLLTs(DL, *EltTy, ValueTys, Offsets,
130 StartingOffset + i * EltSize);
133 // Interpret void as zero return values.
136 // Base case: we can get an LLT for this LLVM IR type.
137 ValueTys.push_back(getLLTForType(Ty, DL));
138 if (Offsets != nullptr)
139 Offsets->push_back(StartingOffset * 8);
142 IRTranslator::ValueToVRegInfo::VRegListT &
143 IRTranslator::allocateVRegs(const Value &Val) {
144 assert(!VMap.contains(Val) && "Value already allocated in VMap");
145 auto *Regs = VMap.getVRegs(Val);
146 auto *Offsets = VMap.getOffsets(Val);
147 SmallVector<LLT, 4> SplitTys;
148 computeValueLLTs(*DL, *Val.getType(), SplitTys,
149 Offsets->empty() ? Offsets : nullptr);
150 for (unsigned i = 0; i < SplitTys.size(); ++i)
155 ArrayRef<unsigned> IRTranslator::getOrCreateVRegs(const Value &Val) {
156 auto VRegsIt = VMap.findVRegs(Val);
157 if (VRegsIt != VMap.vregs_end())
158 return *VRegsIt->second;
160 if (Val.getType()->isVoidTy())
161 return *VMap.getVRegs(Val);
163 // Create entry for this type.
164 auto *VRegs = VMap.getVRegs(Val);
165 auto *Offsets = VMap.getOffsets(Val);
167 assert(Val.getType()->isSized() &&
168 "Don't know how to create an empty vreg");
170 SmallVector<LLT, 4> SplitTys;
171 computeValueLLTs(*DL, *Val.getType(), SplitTys,
172 Offsets->empty() ? Offsets : nullptr);
174 if (!isa<Constant>(Val)) {
175 for (auto Ty : SplitTys)
176 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
180 if (Val.getType()->isAggregateType()) {
181 // UndefValue, ConstantAggregateZero
182 auto &C = cast<Constant>(Val);
184 while (auto Elt = C.getAggregateElement(Idx++)) {
185 auto EltRegs = getOrCreateVRegs(*Elt);
186 std::copy(EltRegs.begin(), EltRegs.end(), std::back_inserter(*VRegs));
189 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
190 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
191 bool Success = translate(cast<Constant>(Val), VRegs->front());
193 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
194 MF->getFunction().getSubprogram(),
195 &MF->getFunction().getEntryBlock());
196 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
197 reportTranslationError(*MF, *TPC, *ORE, R);
205 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
206 if (FrameIndices.find(&AI) != FrameIndices.end())
207 return FrameIndices[&AI];
209 unsigned ElementSize = DL->getTypeStoreSize(AI.getAllocatedType());
211 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
213 // Always allocate at least one byte.
214 Size = std::max(Size, 1u);
216 unsigned Alignment = AI.getAlignment();
218 Alignment = DL->getABITypeAlignment(AI.getAllocatedType());
220 int &FI = FrameIndices[&AI];
221 FI = MF->getFrameInfo().CreateStackObject(Size, Alignment, false, &AI);
225 unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
226 unsigned Alignment = 0;
227 Type *ValTy = nullptr;
228 if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
229 Alignment = SI->getAlignment();
230 ValTy = SI->getValueOperand()->getType();
231 } else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
232 Alignment = LI->getAlignment();
233 ValTy = LI->getType();
234 } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
235 // TODO(PR27168): This instruction has no alignment attribute, but unlike
236 // the default alignment for load/store, the default here is to assume
237 // it has NATURAL alignment, not DataLayout-specified alignment.
238 const DataLayout &DL = AI->getModule()->getDataLayout();
239 Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
240 ValTy = AI->getCompareOperand()->getType();
241 } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
242 // TODO(PR27168): This instruction has no alignment attribute, but unlike
243 // the default alignment for load/store, the default here is to assume
244 // it has NATURAL alignment, not DataLayout-specified alignment.
245 const DataLayout &DL = AI->getModule()->getDataLayout();
246 Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
247 ValTy = AI->getType();
249 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
250 R << "unable to translate memop: " << ore::NV("Opcode", &I);
251 reportTranslationError(*MF, *TPC, *ORE, R);
255 return Alignment ? Alignment : DL->getABITypeAlignment(ValTy);
258 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
259 MachineBasicBlock *&MBB = BBToMBB[&BB];
260 assert(MBB && "BasicBlock was not encountered before");
264 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
265 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
266 MachinePreds[Edge].push_back(NewPred);
269 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
270 MachineIRBuilder &MIRBuilder) {
271 // FIXME: handle signed/unsigned wrapping flags.
273 // Get or create a virtual register for each value.
274 // Unless the value is a Constant => loadimm cst?
275 // or inline constant each time?
276 // Creation of a virtual register needs to have a size.
277 unsigned Op0 = getOrCreateVReg(*U.getOperand(0));
278 unsigned Op1 = getOrCreateVReg(*U.getOperand(1));
279 unsigned Res = getOrCreateVReg(U);
280 MIRBuilder.buildInstr(Opcode).addDef(Res).addUse(Op0).addUse(Op1);
284 bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
285 // -0.0 - X --> G_FNEG
286 if (isa<Constant>(U.getOperand(0)) &&
287 U.getOperand(0) == ConstantFP::getZeroValueForNegation(U.getType())) {
288 MIRBuilder.buildInstr(TargetOpcode::G_FNEG)
289 .addDef(getOrCreateVReg(U))
290 .addUse(getOrCreateVReg(*U.getOperand(1)));
293 return translateBinaryOp(TargetOpcode::G_FSUB, U, MIRBuilder);
296 bool IRTranslator::translateCompare(const User &U,
297 MachineIRBuilder &MIRBuilder) {
298 const CmpInst *CI = dyn_cast<CmpInst>(&U);
299 unsigned Op0 = getOrCreateVReg(*U.getOperand(0));
300 unsigned Op1 = getOrCreateVReg(*U.getOperand(1));
301 unsigned Res = getOrCreateVReg(U);
302 CmpInst::Predicate Pred =
303 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
304 cast<ConstantExpr>(U).getPredicate());
305 if (CmpInst::isIntPredicate(Pred))
306 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
307 else if (Pred == CmpInst::FCMP_FALSE)
308 MIRBuilder.buildCopy(
309 Res, getOrCreateVReg(*Constant::getNullValue(CI->getType())));
310 else if (Pred == CmpInst::FCMP_TRUE)
311 MIRBuilder.buildCopy(
312 Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType())));
314 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1);
319 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
320 const ReturnInst &RI = cast<ReturnInst>(U);
321 const Value *Ret = RI.getReturnValue();
322 if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
324 // The target may mess up with the insertion point, but
325 // this is not important as a return is the last instruction
326 // of the block anyway.
328 // FIXME: this interface should simplify when CallLowering gets adapted to
329 // multiple VRegs per Value.
330 unsigned VReg = Ret ? packRegs(*Ret, MIRBuilder) : 0;
331 return CLI->lowerReturn(MIRBuilder, Ret, VReg);
334 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
335 const BranchInst &BrInst = cast<BranchInst>(U);
337 if (!BrInst.isUnconditional()) {
338 // We want a G_BRCOND to the true BB followed by an unconditional branch.
339 unsigned Tst = getOrCreateVReg(*BrInst.getCondition());
340 const BasicBlock &TrueTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ++));
341 MachineBasicBlock &TrueBB = getMBB(TrueTgt);
342 MIRBuilder.buildBrCond(Tst, TrueBB);
345 const BasicBlock &BrTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ));
346 MachineBasicBlock &TgtBB = getMBB(BrTgt);
347 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
349 // If the unconditional target is the layout successor, fallthrough.
350 if (!CurBB.isLayoutSuccessor(&TgtBB))
351 MIRBuilder.buildBr(TgtBB);
354 for (const BasicBlock *Succ : BrInst.successors())
355 CurBB.addSuccessor(&getMBB(*Succ));
359 bool IRTranslator::translateSwitch(const User &U,
360 MachineIRBuilder &MIRBuilder) {
361 // For now, just translate as a chain of conditional branches.
362 // FIXME: could we share most of the logic/code in
363 // SelectionDAGBuilder::visitSwitch between SelectionDAG and GlobalISel?
364 // At first sight, it seems most of the logic in there is independent of
365 // SelectionDAG-specifics and a lot of work went in to optimize switch
366 // lowering in there.
368 const SwitchInst &SwInst = cast<SwitchInst>(U);
369 const unsigned SwCondValue = getOrCreateVReg(*SwInst.getCondition());
370 const BasicBlock *OrigBB = SwInst.getParent();
372 LLT LLTi1 = getLLTForType(*Type::getInt1Ty(U.getContext()), *DL);
373 for (auto &CaseIt : SwInst.cases()) {
374 const unsigned CaseValueReg = getOrCreateVReg(*CaseIt.getCaseValue());
375 const unsigned Tst = MRI->createGenericVirtualRegister(LLTi1);
376 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, Tst, CaseValueReg, SwCondValue);
377 MachineBasicBlock &CurMBB = MIRBuilder.getMBB();
378 const BasicBlock *TrueBB = CaseIt.getCaseSuccessor();
379 MachineBasicBlock &TrueMBB = getMBB(*TrueBB);
381 MIRBuilder.buildBrCond(Tst, TrueMBB);
382 CurMBB.addSuccessor(&TrueMBB);
383 addMachineCFGPred({OrigBB, TrueBB}, &CurMBB);
385 MachineBasicBlock *FalseMBB =
386 MF->CreateMachineBasicBlock(SwInst.getParent());
387 // Insert the comparison blocks one after the other.
388 MF->insert(std::next(CurMBB.getIterator()), FalseMBB);
389 MIRBuilder.buildBr(*FalseMBB);
390 CurMBB.addSuccessor(FalseMBB);
392 MIRBuilder.setMBB(*FalseMBB);
394 // handle default case
395 const BasicBlock *DefaultBB = SwInst.getDefaultDest();
396 MachineBasicBlock &DefaultMBB = getMBB(*DefaultBB);
397 MIRBuilder.buildBr(DefaultMBB);
398 MachineBasicBlock &CurMBB = MIRBuilder.getMBB();
399 CurMBB.addSuccessor(&DefaultMBB);
400 addMachineCFGPred({OrigBB, DefaultBB}, &CurMBB);
405 bool IRTranslator::translateIndirectBr(const User &U,
406 MachineIRBuilder &MIRBuilder) {
407 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
409 const unsigned Tgt = getOrCreateVReg(*BrInst.getAddress());
410 MIRBuilder.buildBrIndirect(Tgt);
413 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
414 for (const BasicBlock *Succ : BrInst.successors())
415 CurBB.addSuccessor(&getMBB(*Succ));
420 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
421 const LoadInst &LI = cast<LoadInst>(U);
423 auto Flags = LI.isVolatile() ? MachineMemOperand::MOVolatile
424 : MachineMemOperand::MONone;
425 Flags |= MachineMemOperand::MOLoad;
427 if (DL->getTypeStoreSize(LI.getType()) == 0)
430 ArrayRef<unsigned> Regs = getOrCreateVRegs(LI);
431 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
432 unsigned Base = getOrCreateVReg(*LI.getPointerOperand());
434 for (unsigned i = 0; i < Regs.size(); ++i) {
436 MIRBuilder.materializeGEP(Addr, Base, LLT::scalar(64), Offsets[i] / 8);
438 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
439 unsigned BaseAlign = getMemOpAlignment(LI);
440 auto MMO = MF->getMachineMemOperand(
441 Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
442 MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
443 LI.getSyncScopeID(), LI.getOrdering());
444 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
450 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
451 const StoreInst &SI = cast<StoreInst>(U);
452 auto Flags = SI.isVolatile() ? MachineMemOperand::MOVolatile
453 : MachineMemOperand::MONone;
454 Flags |= MachineMemOperand::MOStore;
456 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
459 ArrayRef<unsigned> Vals = getOrCreateVRegs(*SI.getValueOperand());
460 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
461 unsigned Base = getOrCreateVReg(*SI.getPointerOperand());
463 for (unsigned i = 0; i < Vals.size(); ++i) {
465 MIRBuilder.materializeGEP(Addr, Base, LLT::scalar(64), Offsets[i] / 8);
467 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
468 unsigned BaseAlign = getMemOpAlignment(SI);
469 auto MMO = MF->getMachineMemOperand(
470 Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
471 MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
472 SI.getSyncScopeID(), SI.getOrdering());
473 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
478 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
479 const Value *Src = U.getOperand(0);
480 Type *Int32Ty = Type::getInt32Ty(U.getContext());
482 // getIndexedOffsetInType is designed for GEPs, so the first index is the
483 // usual array element rather than looking into the actual aggregate.
484 SmallVector<Value *, 1> Indices;
485 Indices.push_back(ConstantInt::get(Int32Ty, 0));
487 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
488 for (auto Idx : EVI->indices())
489 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
490 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
491 for (auto Idx : IVI->indices())
492 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
494 for (unsigned i = 1; i < U.getNumOperands(); ++i)
495 Indices.push_back(U.getOperand(i));
498 return 8 * static_cast<uint64_t>(
499 DL.getIndexedOffsetInType(Src->getType(), Indices));
502 bool IRTranslator::translateExtractValue(const User &U,
503 MachineIRBuilder &MIRBuilder) {
504 const Value *Src = U.getOperand(0);
505 uint64_t Offset = getOffsetFromIndices(U, *DL);
506 ArrayRef<unsigned> SrcRegs = getOrCreateVRegs(*Src);
507 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
508 unsigned Idx = std::lower_bound(Offsets.begin(), Offsets.end(), Offset) -
510 auto &DstRegs = allocateVRegs(U);
512 for (unsigned i = 0; i < DstRegs.size(); ++i)
513 DstRegs[i] = SrcRegs[Idx++];
518 bool IRTranslator::translateInsertValue(const User &U,
519 MachineIRBuilder &MIRBuilder) {
520 const Value *Src = U.getOperand(0);
521 uint64_t Offset = getOffsetFromIndices(U, *DL);
522 auto &DstRegs = allocateVRegs(U);
523 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
524 ArrayRef<unsigned> SrcRegs = getOrCreateVRegs(*Src);
525 ArrayRef<unsigned> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
526 auto InsertedIt = InsertedRegs.begin();
528 for (unsigned i = 0; i < DstRegs.size(); ++i) {
529 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
530 DstRegs[i] = *InsertedIt++;
532 DstRegs[i] = SrcRegs[i];
538 bool IRTranslator::translateSelect(const User &U,
539 MachineIRBuilder &MIRBuilder) {
540 unsigned Tst = getOrCreateVReg(*U.getOperand(0));
541 ArrayRef<unsigned> ResRegs = getOrCreateVRegs(U);
542 ArrayRef<unsigned> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
543 ArrayRef<unsigned> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
545 for (unsigned i = 0; i < ResRegs.size(); ++i)
546 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i]);
551 bool IRTranslator::translateBitCast(const User &U,
552 MachineIRBuilder &MIRBuilder) {
553 // If we're bitcasting to the source type, we can reuse the source vreg.
554 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
555 getLLTForType(*U.getType(), *DL)) {
556 unsigned SrcReg = getOrCreateVReg(*U.getOperand(0));
557 auto &Regs = *VMap.getVRegs(U);
558 // If we already assigned a vreg for this bitcast, we can't change that.
559 // Emit a copy to satisfy the users we already emitted.
561 MIRBuilder.buildCopy(Regs[0], SrcReg);
563 Regs.push_back(SrcReg);
564 VMap.getOffsets(U)->push_back(0);
568 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
571 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
572 MachineIRBuilder &MIRBuilder) {
573 unsigned Op = getOrCreateVReg(*U.getOperand(0));
574 unsigned Res = getOrCreateVReg(U);
575 MIRBuilder.buildInstr(Opcode).addDef(Res).addUse(Op);
579 bool IRTranslator::translateGetElementPtr(const User &U,
580 MachineIRBuilder &MIRBuilder) {
581 // FIXME: support vector GEPs.
582 if (U.getType()->isVectorTy())
585 Value &Op0 = *U.getOperand(0);
586 unsigned BaseReg = getOrCreateVReg(Op0);
587 Type *PtrIRTy = Op0.getType();
588 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
589 Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
590 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
593 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
595 const Value *Idx = GTI.getOperand();
596 if (StructType *StTy = GTI.getStructTypeOrNull()) {
597 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
598 Offset += DL->getStructLayout(StTy)->getElementOffset(Field);
601 uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
603 // If this is a scalar constant or a splat vector of constants,
604 // handle it quickly.
605 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
606 Offset += ElementSize * CI->getSExtValue();
611 unsigned NewBaseReg = MRI->createGenericVirtualRegister(PtrTy);
613 getOrCreateVReg(*ConstantInt::get(OffsetIRTy, Offset));
614 MIRBuilder.buildGEP(NewBaseReg, BaseReg, OffsetReg);
616 BaseReg = NewBaseReg;
620 unsigned IdxReg = getOrCreateVReg(*Idx);
621 if (MRI->getType(IdxReg) != OffsetTy) {
622 unsigned NewIdxReg = MRI->createGenericVirtualRegister(OffsetTy);
623 MIRBuilder.buildSExtOrTrunc(NewIdxReg, IdxReg);
627 // N = N + Idx * ElementSize;
628 // Avoid doing it for ElementSize of 1.
629 unsigned GepOffsetReg;
630 if (ElementSize != 1) {
631 unsigned ElementSizeReg =
632 getOrCreateVReg(*ConstantInt::get(OffsetIRTy, ElementSize));
634 GepOffsetReg = MRI->createGenericVirtualRegister(OffsetTy);
635 MIRBuilder.buildMul(GepOffsetReg, ElementSizeReg, IdxReg);
637 GepOffsetReg = IdxReg;
639 unsigned NewBaseReg = MRI->createGenericVirtualRegister(PtrTy);
640 MIRBuilder.buildGEP(NewBaseReg, BaseReg, GepOffsetReg);
641 BaseReg = NewBaseReg;
646 unsigned OffsetReg = getOrCreateVReg(*ConstantInt::get(OffsetIRTy, Offset));
647 MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetReg);
651 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
655 bool IRTranslator::translateMemfunc(const CallInst &CI,
656 MachineIRBuilder &MIRBuilder,
658 LLT SizeTy = getLLTForType(*CI.getArgOperand(2)->getType(), *DL);
659 Type *DstTy = CI.getArgOperand(0)->getType();
660 if (cast<PointerType>(DstTy)->getAddressSpace() != 0 ||
661 SizeTy.getSizeInBits() != DL->getPointerSizeInBits(0))
664 SmallVector<CallLowering::ArgInfo, 8> Args;
665 for (int i = 0; i < 3; ++i) {
666 const auto &Arg = CI.getArgOperand(i);
667 Args.emplace_back(getOrCreateVReg(*Arg), Arg->getType());
672 case Intrinsic::memmove:
673 case Intrinsic::memcpy: {
674 Type *SrcTy = CI.getArgOperand(1)->getType();
675 if(cast<PointerType>(SrcTy)->getAddressSpace() != 0)
677 Callee = ID == Intrinsic::memcpy ? "memcpy" : "memmove";
680 case Intrinsic::memset:
687 return CLI->lowerCall(MIRBuilder, CI.getCallingConv(),
688 MachineOperand::CreateES(Callee),
689 CallLowering::ArgInfo(0, CI.getType()), Args);
692 void IRTranslator::getStackGuard(unsigned DstReg,
693 MachineIRBuilder &MIRBuilder) {
694 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
695 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
696 auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD);
699 auto &TLI = *MF->getSubtarget().getTargetLowering();
700 Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
704 MachinePointerInfo MPInfo(Global);
705 MachineInstr::mmo_iterator MemRefs = MF->allocateMemRefsArray(1);
706 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
707 MachineMemOperand::MODereferenceable;
709 MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
710 DL->getPointerABIAlignment(0));
711 MIB.setMemRefs(MemRefs, MemRefs + 1);
714 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
715 MachineIRBuilder &MIRBuilder) {
716 ArrayRef<unsigned> ResRegs = getOrCreateVRegs(CI);
717 auto MIB = MIRBuilder.buildInstr(Op)
720 .addUse(getOrCreateVReg(*CI.getOperand(0)))
721 .addUse(getOrCreateVReg(*CI.getOperand(1)));
723 if (Op == TargetOpcode::G_UADDE || Op == TargetOpcode::G_USUBE) {
724 unsigned Zero = getOrCreateVReg(
725 *Constant::getNullValue(Type::getInt1Ty(CI.getContext())));
732 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
733 MachineIRBuilder &MIRBuilder) {
737 case Intrinsic::lifetime_start:
738 case Intrinsic::lifetime_end:
739 // Stack coloring is not enabled in O0 (which we care about now) so we can
740 // drop these. Make sure someone notices when we start compiling at higher
742 if (MF->getTarget().getOptLevel() != CodeGenOpt::None)
745 case Intrinsic::dbg_declare: {
746 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
747 assert(DI.getVariable() && "Missing variable");
749 const Value *Address = DI.getAddress();
750 if (!Address || isa<UndefValue>(Address)) {
751 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
755 assert(DI.getVariable()->isValidLocationForIntrinsic(
756 MIRBuilder.getDebugLoc()) &&
757 "Expected inlined-at fields to agree");
758 auto AI = dyn_cast<AllocaInst>(Address);
759 if (AI && AI->isStaticAlloca()) {
760 // Static allocas are tracked at the MF level, no need for DBG_VALUE
761 // instructions (in fact, they get ignored if they *do* exist).
762 MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
763 getOrCreateFrameIndex(*AI), DI.getDebugLoc());
765 MIRBuilder.buildDirectDbgValue(getOrCreateVReg(*Address),
766 DI.getVariable(), DI.getExpression());
769 case Intrinsic::vaend:
770 // No target I know of cares about va_end. Certainly no in-tree target
771 // does. Simplest intrinsic ever!
773 case Intrinsic::vastart: {
774 auto &TLI = *MF->getSubtarget().getTargetLowering();
775 Value *Ptr = CI.getArgOperand(0);
776 unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
778 MIRBuilder.buildInstr(TargetOpcode::G_VASTART)
779 .addUse(getOrCreateVReg(*Ptr))
780 .addMemOperand(MF->getMachineMemOperand(
781 MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 0));
784 case Intrinsic::dbg_value: {
785 // This form of DBG_VALUE is target-independent.
786 const DbgValueInst &DI = cast<DbgValueInst>(CI);
787 const Value *V = DI.getValue();
788 assert(DI.getVariable()->isValidLocationForIntrinsic(
789 MIRBuilder.getDebugLoc()) &&
790 "Expected inlined-at fields to agree");
792 // Currently the optimizer can produce this; insert an undef to
793 // help debugging. Probably the optimizer should not do this.
794 MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
795 } else if (const auto *CI = dyn_cast<Constant>(V)) {
796 MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
798 unsigned Reg = getOrCreateVReg(*V);
799 // FIXME: This does not handle register-indirect values at offset 0. The
800 // direct/indirect thing shouldn't really be handled by something as
801 // implicit as reg+noreg vs reg+imm in the first palce, but it seems
802 // pretty baked in right now.
803 MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
807 case Intrinsic::uadd_with_overflow:
808 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDE, MIRBuilder);
809 case Intrinsic::sadd_with_overflow:
810 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
811 case Intrinsic::usub_with_overflow:
812 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBE, MIRBuilder);
813 case Intrinsic::ssub_with_overflow:
814 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
815 case Intrinsic::umul_with_overflow:
816 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
817 case Intrinsic::smul_with_overflow:
818 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
820 MIRBuilder.buildInstr(TargetOpcode::G_FPOW)
821 .addDef(getOrCreateVReg(CI))
822 .addUse(getOrCreateVReg(*CI.getArgOperand(0)))
823 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
826 MIRBuilder.buildInstr(TargetOpcode::G_FEXP)
827 .addDef(getOrCreateVReg(CI))
828 .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
830 case Intrinsic::exp2:
831 MIRBuilder.buildInstr(TargetOpcode::G_FEXP2)
832 .addDef(getOrCreateVReg(CI))
833 .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
836 MIRBuilder.buildInstr(TargetOpcode::G_FLOG)
837 .addDef(getOrCreateVReg(CI))
838 .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
840 case Intrinsic::log2:
841 MIRBuilder.buildInstr(TargetOpcode::G_FLOG2)
842 .addDef(getOrCreateVReg(CI))
843 .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
845 case Intrinsic::fabs:
846 MIRBuilder.buildInstr(TargetOpcode::G_FABS)
847 .addDef(getOrCreateVReg(CI))
848 .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
851 MIRBuilder.buildInstr(TargetOpcode::G_FMA)
852 .addDef(getOrCreateVReg(CI))
853 .addUse(getOrCreateVReg(*CI.getArgOperand(0)))
854 .addUse(getOrCreateVReg(*CI.getArgOperand(1)))
855 .addUse(getOrCreateVReg(*CI.getArgOperand(2)));
857 case Intrinsic::fmuladd: {
858 const TargetMachine &TM = MF->getTarget();
859 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
860 unsigned Dst = getOrCreateVReg(CI);
861 unsigned Op0 = getOrCreateVReg(*CI.getArgOperand(0));
862 unsigned Op1 = getOrCreateVReg(*CI.getArgOperand(1));
863 unsigned Op2 = getOrCreateVReg(*CI.getArgOperand(2));
864 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
865 TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) {
866 // TODO: Revisit this to see if we should move this part of the
867 // lowering to the combiner.
868 MIRBuilder.buildInstr(TargetOpcode::G_FMA, Dst, Op0, Op1, Op2);
870 LLT Ty = getLLTForType(*CI.getType(), *DL);
871 auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, Ty, Op0, Op1);
872 MIRBuilder.buildInstr(TargetOpcode::G_FADD, Dst, FMul, Op2);
876 case Intrinsic::memcpy:
877 case Intrinsic::memmove:
878 case Intrinsic::memset:
879 return translateMemfunc(CI, MIRBuilder, ID);
880 case Intrinsic::eh_typeid_for: {
881 GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));
882 unsigned Reg = getOrCreateVReg(CI);
883 unsigned TypeID = MF->getTypeIDFor(GV);
884 MIRBuilder.buildConstant(Reg, TypeID);
887 case Intrinsic::objectsize: {
888 // If we don't know by now, we're never going to know.
889 const ConstantInt *Min = cast<ConstantInt>(CI.getArgOperand(1));
891 MIRBuilder.buildConstant(getOrCreateVReg(CI), Min->isZero() ? -1ULL : 0);
894 case Intrinsic::stackguard:
895 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
897 case Intrinsic::stackprotector: {
898 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
899 unsigned GuardVal = MRI->createGenericVirtualRegister(PtrTy);
900 getStackGuard(GuardVal, MIRBuilder);
902 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
903 MIRBuilder.buildStore(
904 GuardVal, getOrCreateVReg(*Slot),
905 *MF->getMachineMemOperand(
906 MachinePointerInfo::getFixedStack(*MF,
907 getOrCreateFrameIndex(*Slot)),
908 MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
909 PtrTy.getSizeInBits() / 8, 8));
916 bool IRTranslator::translateInlineAsm(const CallInst &CI,
917 MachineIRBuilder &MIRBuilder) {
918 const InlineAsm &IA = cast<InlineAsm>(*CI.getCalledValue());
919 if (!IA.getConstraintString().empty())
922 unsigned ExtraInfo = 0;
923 if (IA.hasSideEffects())
924 ExtraInfo |= InlineAsm::Extra_HasSideEffects;
925 if (IA.getDialect() == InlineAsm::AD_Intel)
926 ExtraInfo |= InlineAsm::Extra_AsmDialect;
928 MIRBuilder.buildInstr(TargetOpcode::INLINEASM)
929 .addExternalSymbol(IA.getAsmString().c_str())
935 unsigned IRTranslator::packRegs(const Value &V,
936 MachineIRBuilder &MIRBuilder) {
937 ArrayRef<unsigned> Regs = getOrCreateVRegs(V);
938 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V);
939 LLT BigTy = getLLTForType(*V.getType(), *DL);
941 if (Regs.size() == 1)
944 unsigned Dst = MRI->createGenericVirtualRegister(BigTy);
945 MIRBuilder.buildUndef(Dst);
946 for (unsigned i = 0; i < Regs.size(); ++i) {
947 unsigned NewDst = MRI->createGenericVirtualRegister(BigTy);
948 MIRBuilder.buildInsert(NewDst, Dst, Regs[i], Offsets[i]);
954 void IRTranslator::unpackRegs(const Value &V, unsigned Src,
955 MachineIRBuilder &MIRBuilder) {
956 ArrayRef<unsigned> Regs = getOrCreateVRegs(V);
957 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(V);
959 for (unsigned i = 0; i < Regs.size(); ++i)
960 MIRBuilder.buildExtract(Regs[i], Src, Offsets[i]);
963 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
964 const CallInst &CI = cast<CallInst>(U);
965 auto TII = MF->getTarget().getIntrinsicInfo();
966 const Function *F = CI.getCalledFunction();
968 // FIXME: support Windows dllimport function calls.
969 if (F && F->hasDLLImportStorageClass())
972 if (CI.isInlineAsm())
973 return translateInlineAsm(CI, MIRBuilder);
975 Intrinsic::ID ID = Intrinsic::not_intrinsic;
976 if (F && F->isIntrinsic()) {
977 ID = F->getIntrinsicID();
978 if (TII && ID == Intrinsic::not_intrinsic)
979 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
982 bool IsSplitType = valueIsSplit(CI);
983 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic) {
984 unsigned Res = IsSplitType ? MRI->createGenericVirtualRegister(
985 getLLTForType(*CI.getType(), *DL))
986 : getOrCreateVReg(CI);
988 SmallVector<unsigned, 8> Args;
989 for (auto &Arg: CI.arg_operands())
990 Args.push_back(packRegs(*Arg, MIRBuilder));
992 MF->getFrameInfo().setHasCalls(true);
993 bool Success = CLI->lowerCall(MIRBuilder, &CI, Res, Args, [&]() {
994 return getOrCreateVReg(*CI.getCalledValue());
998 unpackRegs(CI, Res, MIRBuilder);
1002 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
1004 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
1008 if (!CI.getType()->isVoidTy()) {
1011 MRI->createGenericVirtualRegister(getLLTForType(*CI.getType(), *DL));
1013 Res = getOrCreateVReg(CI);
1015 MachineInstrBuilder MIB =
1016 MIRBuilder.buildIntrinsic(ID, Res, !CI.doesNotAccessMemory());
1018 for (auto &Arg : CI.arg_operands()) {
1019 // Some intrinsics take metadata parameters. Reject them.
1020 if (isa<MetadataAsValue>(Arg))
1022 MIB.addUse(packRegs(*Arg, MIRBuilder));
1026 unpackRegs(CI, Res, MIRBuilder);
1028 // Add a MachineMemOperand if it is a target mem intrinsic.
1029 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1030 TargetLowering::IntrinsicInfo Info;
1031 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
1032 if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
1033 uint64_t Size = Info.memVT.getStoreSize();
1034 MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
1035 Info.flags, Size, Info.align));
1041 bool IRTranslator::translateInvoke(const User &U,
1042 MachineIRBuilder &MIRBuilder) {
1043 const InvokeInst &I = cast<InvokeInst>(U);
1044 MCContext &Context = MF->getContext();
1046 const BasicBlock *ReturnBB = I.getSuccessor(0);
1047 const BasicBlock *EHPadBB = I.getSuccessor(1);
1049 const Value *Callee = I.getCalledValue();
1050 const Function *Fn = dyn_cast<Function>(Callee);
1051 if (isa<InlineAsm>(Callee))
1054 // FIXME: support invoking patchpoint and statepoint intrinsics.
1055 if (Fn && Fn->isIntrinsic())
1058 // FIXME: support whatever these are.
1059 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
1062 // FIXME: support Windows exception handling.
1063 if (!isa<LandingPadInst>(EHPadBB->front()))
1066 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
1067 // the region covered by the try.
1068 MCSymbol *BeginSymbol = Context.createTempSymbol();
1069 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
1072 MRI->createGenericVirtualRegister(getLLTForType(*I.getType(), *DL));
1073 SmallVector<unsigned, 8> Args;
1074 for (auto &Arg: I.arg_operands())
1075 Args.push_back(packRegs(*Arg, MIRBuilder));
1077 if (!CLI->lowerCall(MIRBuilder, &I, Res, Args,
1078 [&]() { return getOrCreateVReg(*I.getCalledValue()); }))
1081 unpackRegs(I, Res, MIRBuilder);
1083 MCSymbol *EndSymbol = Context.createTempSymbol();
1084 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
1086 // FIXME: track probabilities.
1087 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
1088 &ReturnMBB = getMBB(*ReturnBB);
1089 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
1090 MIRBuilder.getMBB().addSuccessor(&ReturnMBB);
1091 MIRBuilder.getMBB().addSuccessor(&EHPadMBB);
1092 MIRBuilder.buildBr(ReturnMBB);
1097 bool IRTranslator::translateLandingPad(const User &U,
1098 MachineIRBuilder &MIRBuilder) {
1099 const LandingPadInst &LP = cast<LandingPadInst>(U);
1101 MachineBasicBlock &MBB = MIRBuilder.getMBB();
1102 addLandingPadInfo(LP, MBB);
1106 // If there aren't registers to copy the values into (e.g., during SjLj
1107 // exceptions), then don't bother.
1108 auto &TLI = *MF->getSubtarget().getTargetLowering();
1109 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
1110 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
1111 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
1114 // If landingpad's return type is token type, we don't create DAG nodes
1115 // for its exception pointer and selector value. The extraction of exception
1116 // pointer or selector value from token type landingpads is not currently
1118 if (LP.getType()->isTokenTy())
1121 // Add a label to mark the beginning of the landing pad. Deletion of the
1122 // landing pad can thus be detected via the MachineModuleInfo.
1123 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
1124 .addSym(MF->addLandingPad(&MBB));
1126 LLT Ty = getLLTForType(*LP.getType(), *DL);
1127 unsigned Undef = MRI->createGenericVirtualRegister(Ty);
1128 MIRBuilder.buildUndef(Undef);
1130 SmallVector<LLT, 2> Tys;
1131 for (Type *Ty : cast<StructType>(LP.getType())->elements())
1132 Tys.push_back(getLLTForType(*Ty, *DL));
1133 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
1135 // Mark exception register as live in.
1136 unsigned ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
1140 MBB.addLiveIn(ExceptionReg);
1141 ArrayRef<unsigned> ResRegs = getOrCreateVRegs(LP);
1142 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
1144 unsigned SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
1148 MBB.addLiveIn(SelectorReg);
1149 unsigned PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
1150 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
1151 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
1156 bool IRTranslator::translateAlloca(const User &U,
1157 MachineIRBuilder &MIRBuilder) {
1158 auto &AI = cast<AllocaInst>(U);
1160 if (AI.isSwiftError())
1163 if (AI.isStaticAlloca()) {
1164 unsigned Res = getOrCreateVReg(AI);
1165 int FI = getOrCreateFrameIndex(AI);
1166 MIRBuilder.buildFrameIndex(Res, FI);
1170 // FIXME: support stack probing for Windows.
1171 if (MF->getTarget().getTargetTriple().isOSWindows())
1174 // Now we're in the harder dynamic case.
1175 Type *Ty = AI.getAllocatedType();
1177 std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment());
1179 unsigned NumElts = getOrCreateVReg(*AI.getArraySize());
1181 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
1182 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
1183 if (MRI->getType(NumElts) != IntPtrTy) {
1184 unsigned ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
1185 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
1189 unsigned AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
1191 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, -DL->getTypeAllocSize(Ty)));
1192 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
1194 LLT PtrTy = getLLTForType(*AI.getType(), *DL);
1195 auto &TLI = *MF->getSubtarget().getTargetLowering();
1196 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1198 unsigned SPTmp = MRI->createGenericVirtualRegister(PtrTy);
1199 MIRBuilder.buildCopy(SPTmp, SPReg);
1201 unsigned AllocTmp = MRI->createGenericVirtualRegister(PtrTy);
1202 MIRBuilder.buildGEP(AllocTmp, SPTmp, AllocSize);
1204 // Handle alignment. We have to realign if the allocation granule was smaller
1205 // than stack alignment, or the specific alloca requires more than stack
1207 unsigned StackAlign =
1208 MF->getSubtarget().getFrameLowering()->getStackAlignment();
1209 Align = std::max(Align, StackAlign);
1210 if (Align > StackAlign || DL->getTypeAllocSize(Ty) % StackAlign != 0) {
1211 // Round the size of the allocation up to the stack alignment size
1212 // by add SA-1 to the size. This doesn't overflow because we're computing
1213 // an address inside an alloca.
1214 unsigned AlignedAlloc = MRI->createGenericVirtualRegister(PtrTy);
1215 MIRBuilder.buildPtrMask(AlignedAlloc, AllocTmp, Log2_32(Align));
1216 AllocTmp = AlignedAlloc;
1219 MIRBuilder.buildCopy(SPReg, AllocTmp);
1220 MIRBuilder.buildCopy(getOrCreateVReg(AI), AllocTmp);
1222 MF->getFrameInfo().CreateVariableSizedObject(Align ? Align : 1, &AI);
1223 assert(MF->getFrameInfo().hasVarSizedObjects());
1227 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
1228 // FIXME: We may need more info about the type. Because of how LLT works,
1229 // we're completely discarding the i64/double distinction here (amongst
1230 // others). Fortunately the ABIs I know of where that matters don't use va_arg
1231 // anyway but that's not guaranteed.
1232 MIRBuilder.buildInstr(TargetOpcode::G_VAARG)
1233 .addDef(getOrCreateVReg(U))
1234 .addUse(getOrCreateVReg(*U.getOperand(0)))
1235 .addImm(DL->getABITypeAlignment(U.getType()));
1239 bool IRTranslator::translateInsertElement(const User &U,
1240 MachineIRBuilder &MIRBuilder) {
1241 // If it is a <1 x Ty> vector, use the scalar as it is
1242 // not a legal vector type in LLT.
1243 if (U.getType()->getVectorNumElements() == 1) {
1244 unsigned Elt = getOrCreateVReg(*U.getOperand(1));
1245 auto &Regs = *VMap.getVRegs(U);
1247 Regs.push_back(Elt);
1248 VMap.getOffsets(U)->push_back(0);
1250 MIRBuilder.buildCopy(Regs[0], Elt);
1255 unsigned Res = getOrCreateVReg(U);
1256 unsigned Val = getOrCreateVReg(*U.getOperand(0));
1257 unsigned Elt = getOrCreateVReg(*U.getOperand(1));
1258 unsigned Idx = getOrCreateVReg(*U.getOperand(2));
1259 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
1263 bool IRTranslator::translateExtractElement(const User &U,
1264 MachineIRBuilder &MIRBuilder) {
1265 // If it is a <1 x Ty> vector, use the scalar as it is
1266 // not a legal vector type in LLT.
1267 if (U.getOperand(0)->getType()->getVectorNumElements() == 1) {
1268 unsigned Elt = getOrCreateVReg(*U.getOperand(0));
1269 auto &Regs = *VMap.getVRegs(U);
1271 Regs.push_back(Elt);
1272 VMap.getOffsets(U)->push_back(0);
1274 MIRBuilder.buildCopy(Regs[0], Elt);
1278 unsigned Res = getOrCreateVReg(U);
1279 unsigned Val = getOrCreateVReg(*U.getOperand(0));
1280 unsigned Idx = getOrCreateVReg(*U.getOperand(1));
1281 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
1285 bool IRTranslator::translateShuffleVector(const User &U,
1286 MachineIRBuilder &MIRBuilder) {
1287 MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
1288 .addDef(getOrCreateVReg(U))
1289 .addUse(getOrCreateVReg(*U.getOperand(0)))
1290 .addUse(getOrCreateVReg(*U.getOperand(1)))
1291 .addUse(getOrCreateVReg(*U.getOperand(2)));
1295 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
1296 const PHINode &PI = cast<PHINode>(U);
1298 SmallVector<MachineInstr *, 4> Insts;
1299 for (auto Reg : getOrCreateVRegs(PI)) {
1300 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, Reg);
1301 Insts.push_back(MIB.getInstr());
1304 PendingPHIs.emplace_back(&PI, std::move(Insts));
1308 bool IRTranslator::translateAtomicCmpXchg(const User &U,
1309 MachineIRBuilder &MIRBuilder) {
1310 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
1315 auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1316 : MachineMemOperand::MONone;
1317 Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1319 Type *ResType = I.getType();
1320 Type *ValType = ResType->Type::getStructElementType(0);
1322 auto Res = getOrCreateVRegs(I);
1323 unsigned OldValRes = Res[0];
1324 unsigned SuccessRes = Res[1];
1325 unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
1326 unsigned Cmp = getOrCreateVReg(*I.getCompareOperand());
1327 unsigned NewVal = getOrCreateVReg(*I.getNewValOperand());
1329 MIRBuilder.buildAtomicCmpXchgWithSuccess(
1330 OldValRes, SuccessRes, Addr, Cmp, NewVal,
1331 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
1332 Flags, DL->getTypeStoreSize(ValType),
1333 getMemOpAlignment(I), AAMDNodes(), nullptr,
1334 I.getSyncScopeID(), I.getSuccessOrdering(),
1335 I.getFailureOrdering()));
1339 bool IRTranslator::translateAtomicRMW(const User &U,
1340 MachineIRBuilder &MIRBuilder) {
1341 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
1343 auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1344 : MachineMemOperand::MONone;
1345 Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1347 Type *ResType = I.getType();
1349 unsigned Res = getOrCreateVReg(I);
1350 unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
1351 unsigned Val = getOrCreateVReg(*I.getValOperand());
1353 unsigned Opcode = 0;
1354 switch (I.getOperation()) {
1356 llvm_unreachable("Unknown atomicrmw op");
1358 case AtomicRMWInst::Xchg:
1359 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
1361 case AtomicRMWInst::Add:
1362 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
1364 case AtomicRMWInst::Sub:
1365 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
1367 case AtomicRMWInst::And:
1368 Opcode = TargetOpcode::G_ATOMICRMW_AND;
1370 case AtomicRMWInst::Nand:
1371 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
1373 case AtomicRMWInst::Or:
1374 Opcode = TargetOpcode::G_ATOMICRMW_OR;
1376 case AtomicRMWInst::Xor:
1377 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
1379 case AtomicRMWInst::Max:
1380 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
1382 case AtomicRMWInst::Min:
1383 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
1385 case AtomicRMWInst::UMax:
1386 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
1388 case AtomicRMWInst::UMin:
1389 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
1393 MIRBuilder.buildAtomicRMW(
1394 Opcode, Res, Addr, Val,
1395 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
1396 Flags, DL->getTypeStoreSize(ResType),
1397 getMemOpAlignment(I), AAMDNodes(), nullptr,
1398 I.getSyncScopeID(), I.getOrdering()));
1402 void IRTranslator::finishPendingPhis() {
1403 for (auto &Phi : PendingPHIs) {
1404 const PHINode *PI = Phi.first;
1405 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
1407 // All MachineBasicBlocks exist, add them to the PHI. We assume IRTranslator
1408 // won't create extra control flow here, otherwise we need to find the
1409 // dominating predecessor here (or perhaps force the weirder IRTranslators
1410 // to provide a simple boundary).
1411 SmallSet<const BasicBlock *, 4> HandledPreds;
1413 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
1414 auto IRPred = PI->getIncomingBlock(i);
1415 if (HandledPreds.count(IRPred))
1418 HandledPreds.insert(IRPred);
1419 ArrayRef<unsigned> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
1420 for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
1421 assert(Pred->isSuccessor(ComponentPHIs[0]->getParent()) &&
1422 "incorrect CFG at MachineBasicBlock level");
1423 for (unsigned j = 0; j < ValRegs.size(); ++j) {
1424 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
1425 MIB.addUse(ValRegs[j]);
1433 bool IRTranslator::valueIsSplit(const Value &V,
1434 SmallVectorImpl<uint64_t> *Offsets) {
1435 SmallVector<LLT, 4> SplitTys;
1436 computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
1437 return SplitTys.size() > 1;
1440 bool IRTranslator::translate(const Instruction &Inst) {
1441 CurBuilder.setDebugLoc(Inst.getDebugLoc());
1442 switch(Inst.getOpcode()) {
1443 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1444 case Instruction::OPCODE: return translate##OPCODE(Inst, CurBuilder);
1445 #include "llvm/IR/Instruction.def"
1451 bool IRTranslator::translate(const Constant &C, unsigned Reg) {
1452 if (auto CI = dyn_cast<ConstantInt>(&C))
1453 EntryBuilder.buildConstant(Reg, *CI);
1454 else if (auto CF = dyn_cast<ConstantFP>(&C))
1455 EntryBuilder.buildFConstant(Reg, *CF);
1456 else if (isa<UndefValue>(C))
1457 EntryBuilder.buildUndef(Reg);
1458 else if (isa<ConstantPointerNull>(C)) {
1459 // As we are trying to build a constant val of 0 into a pointer,
1460 // insert a cast to make them correct with respect to types.
1461 unsigned NullSize = DL->getTypeSizeInBits(C.getType());
1462 auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
1463 auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
1464 unsigned ZeroReg = getOrCreateVReg(*ZeroVal);
1465 EntryBuilder.buildCast(Reg, ZeroReg);
1466 } else if (auto GV = dyn_cast<GlobalValue>(&C))
1467 EntryBuilder.buildGlobalValue(Reg, GV);
1468 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
1469 if (!CAZ->getType()->isVectorTy())
1471 // Return the scalar if it is a <1 x Ty> vector.
1472 if (CAZ->getNumElements() == 1)
1473 return translate(*CAZ->getElementValue(0u), Reg);
1474 std::vector<unsigned> Ops;
1475 for (unsigned i = 0; i < CAZ->getNumElements(); ++i) {
1476 Constant &Elt = *CAZ->getElementValue(i);
1477 Ops.push_back(getOrCreateVReg(Elt));
1479 EntryBuilder.buildMerge(Reg, Ops);
1480 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
1481 // Return the scalar if it is a <1 x Ty> vector.
1482 if (CV->getNumElements() == 1)
1483 return translate(*CV->getElementAsConstant(0), Reg);
1484 std::vector<unsigned> Ops;
1485 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
1486 Constant &Elt = *CV->getElementAsConstant(i);
1487 Ops.push_back(getOrCreateVReg(Elt));
1489 EntryBuilder.buildMerge(Reg, Ops);
1490 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
1491 switch(CE->getOpcode()) {
1492 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1493 case Instruction::OPCODE: return translate##OPCODE(*CE, EntryBuilder);
1494 #include "llvm/IR/Instruction.def"
1498 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
1499 if (CV->getNumOperands() == 1)
1500 return translate(*CV->getOperand(0), Reg);
1501 SmallVector<unsigned, 4> Ops;
1502 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
1503 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
1505 EntryBuilder.buildMerge(Reg, Ops);
1512 void IRTranslator::finalizeFunction() {
1513 // Release the memory used by the different maps we
1514 // needed during the translation.
1515 PendingPHIs.clear();
1517 FrameIndices.clear();
1518 MachinePreds.clear();
1519 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
1520 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
1521 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
1522 EntryBuilder = MachineIRBuilder();
1523 CurBuilder = MachineIRBuilder();
1526 bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
1528 const Function &F = MF->getFunction();
1531 CLI = MF->getSubtarget().getCallLowering();
1532 CurBuilder.setMF(*MF);
1533 EntryBuilder.setMF(*MF);
1534 MRI = &MF->getRegInfo();
1535 DL = &F.getParent()->getDataLayout();
1536 TPC = &getAnalysis<TargetPassConfig>();
1537 ORE = llvm::make_unique<OptimizationRemarkEmitter>(&F);
1539 assert(PendingPHIs.empty() && "stale PHIs");
1541 if (!DL->isLittleEndian()) {
1542 // Currently we don't properly handle big endian code.
1543 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
1544 F.getSubprogram(), &F.getEntryBlock());
1545 R << "unable to translate in big endian mode";
1546 reportTranslationError(*MF, *TPC, *ORE, R);
1549 // Release the per-function state when we return, whether we succeeded or not.
1550 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
1552 // Setup a separate basic-block for the arguments and constants
1553 MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
1554 MF->push_back(EntryBB);
1555 EntryBuilder.setMBB(*EntryBB);
1557 // Create all blocks, in IR order, to preserve the layout.
1558 for (const BasicBlock &BB: F) {
1559 auto *&MBB = BBToMBB[&BB];
1561 MBB = MF->CreateMachineBasicBlock(&BB);
1564 if (BB.hasAddressTaken())
1565 MBB->setHasAddressTaken();
1568 // Make our arguments/constants entry block fallthrough to the IR entry block.
1569 EntryBB->addSuccessor(&getMBB(F.front()));
1571 // Lower the actual args into this basic block.
1572 SmallVector<unsigned, 8> VRegArgs;
1573 for (const Argument &Arg: F.args()) {
1574 if (DL->getTypeStoreSize(Arg.getType()) == 0)
1575 continue; // Don't handle zero sized types.
1577 MRI->createGenericVirtualRegister(getLLTForType(*Arg.getType(), *DL)));
1580 // We don't currently support translating swifterror or swiftself functions.
1581 for (auto &Arg : F.args()) {
1582 if (Arg.hasSwiftErrorAttr() || Arg.hasSwiftSelfAttr()) {
1583 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
1584 F.getSubprogram(), &F.getEntryBlock());
1585 R << "unable to lower arguments due to swifterror/swiftself: "
1586 << ore::NV("Prototype", F.getType());
1587 reportTranslationError(*MF, *TPC, *ORE, R);
1592 if (!CLI->lowerFormalArguments(EntryBuilder, F, VRegArgs)) {
1593 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
1594 F.getSubprogram(), &F.getEntryBlock());
1595 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
1596 reportTranslationError(*MF, *TPC, *ORE, R);
1600 auto ArgIt = F.arg_begin();
1601 for (auto &VArg : VRegArgs) {
1602 // If the argument is an unsplit scalar then don't use unpackRegs to avoid
1603 // creating redundant copies.
1604 if (!valueIsSplit(*ArgIt, VMap.getOffsets(*ArgIt))) {
1605 auto &VRegs = *VMap.getVRegs(cast<Value>(*ArgIt));
1606 assert(VRegs.empty() && "VRegs already populated?");
1607 VRegs.push_back(VArg);
1609 unpackRegs(*ArgIt, VArg, EntryBuilder);
1614 // And translate the function!
1615 for (const BasicBlock &BB : F) {
1616 MachineBasicBlock &MBB = getMBB(BB);
1617 // Set the insertion point of all the following translations to
1618 // the end of this basic block.
1619 CurBuilder.setMBB(MBB);
1621 for (const Instruction &Inst : BB) {
1622 if (translate(Inst))
1625 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
1626 Inst.getDebugLoc(), &BB);
1627 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
1629 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
1630 std::string InstStrStorage;
1631 raw_string_ostream InstStr(InstStrStorage);
1634 R << ": '" << InstStr.str() << "'";
1637 reportTranslationError(*MF, *TPC, *ORE, R);
1642 finishPendingPhis();
1644 // Merge the argument lowering and constants block with its single
1645 // successor, the LLVM-IR entry block. We want the basic block to
1647 assert(EntryBB->succ_size() == 1 &&
1648 "Custom BB used for lowering should have only one successor");
1649 // Get the successor of the current entry block.
1650 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
1651 assert(NewEntryBB.pred_size() == 1 &&
1652 "LLVM-IR entry block has a predecessor!?");
1653 // Move all the instruction from the current entry block to the
1655 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
1658 // Update the live-in information for the new entry block.
1659 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
1660 NewEntryBB.addLiveIn(LiveIn);
1661 NewEntryBB.sortUniqueLiveIns();
1663 // Get rid of the now empty basic block.
1664 EntryBB->removeSuccessor(&NewEntryBB);
1665 MF->remove(EntryBB);
1666 MF->DeleteMachineBasicBlock(EntryBB);
1668 assert(&MF->front() == &NewEntryBB &&
1669 "New entry wasn't next in the list of basic block!");
1671 // Initialize stack protector information.
1672 StackProtector &SP = getAnalysis<StackProtector>();
1673 SP.copyToMachineFrameInfo(MF->getFrameInfo());