1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file implements some simple delegations needed for call lowering.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/CodeGen/Analysis.h"
15 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
16 #include "llvm/CodeGen/GlobalISel/Utils.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/MachineOperand.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/LLVMContext.h"
24 #include "llvm/IR/Module.h"
26 #define DEBUG_TYPE "call-lowering"
30 void CallLowering::anchor() {}
32 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
33 ArrayRef<Register> ResRegs,
34 ArrayRef<ArrayRef<Register>> ArgRegs,
35 Register SwiftErrorVReg,
36 std::function<unsigned()> GetCalleeReg) const {
37 CallLoweringInfo Info;
38 auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout();
40 // First step is to marshall all the function's parameters into the correct
41 // physregs and memory locations. Gather the sequence of argument types that
42 // we'll pass to the assigner function.
44 unsigned NumFixedArgs = CS.getFunctionType()->getNumParams();
45 for (auto &Arg : CS.args()) {
46 ArgInfo OrigArg{ArgRegs[i], Arg->getType(), ISD::ArgFlagsTy{},
48 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CS);
49 Info.OrigArgs.push_back(OrigArg);
53 if (const Function *F = CS.getCalledFunction())
54 Info.Callee = MachineOperand::CreateGA(F, 0);
56 Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
58 Info.OrigRet = ArgInfo{ResRegs, CS.getType(), ISD::ArgFlagsTy{}};
59 if (!Info.OrigRet.Ty->isVoidTy())
60 setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CS);
63 CS.getInstruction()->getMetadata(LLVMContext::MD_callees);
64 Info.CallConv = CS.getCallingConv();
65 Info.SwiftErrorVReg = SwiftErrorVReg;
66 Info.IsMustTailCall = CS.isMustTailCall();
67 Info.IsTailCall = CS.isTailCall() &&
68 isInTailCallPosition(CS, MIRBuilder.getMF().getTarget()) &&
71 .getFnAttribute("disable-tail-calls")
72 .getValueAsString() != "true");
73 Info.IsVarArg = CS.getFunctionType()->isVarArg();
74 return lowerCall(MIRBuilder, Info);
77 template <typename FuncInfoTy>
78 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
80 const FuncInfoTy &FuncInfo) const {
81 auto &Flags = Arg.Flags[0];
82 const AttributeList &Attrs = FuncInfo.getAttributes();
83 if (Attrs.hasAttribute(OpIdx, Attribute::ZExt))
85 if (Attrs.hasAttribute(OpIdx, Attribute::SExt))
87 if (Attrs.hasAttribute(OpIdx, Attribute::InReg))
89 if (Attrs.hasAttribute(OpIdx, Attribute::StructRet))
91 if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf))
93 if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError))
94 Flags.setSwiftError();
95 if (Attrs.hasAttribute(OpIdx, Attribute::ByVal))
97 if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca))
100 if (Flags.isByVal() || Flags.isInAlloca()) {
101 Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
103 auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType();
104 Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy));
106 // For ByVal, alignment should be passed from FE. BE will guess if
107 // this info is not there but there are cases it cannot get right.
109 if (FuncInfo.getParamAlignment(OpIdx - 2))
110 FrameAlign = FuncInfo.getParamAlignment(OpIdx - 2);
112 FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
113 Flags.setByValAlign(Align(FrameAlign));
115 if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
117 Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty)));
121 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
122 const DataLayout &DL,
123 const Function &FuncInfo) const;
126 CallLowering::setArgFlags<CallInst>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
127 const DataLayout &DL,
128 const CallInst &FuncInfo) const;
130 Register CallLowering::packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy,
131 MachineIRBuilder &MIRBuilder) const {
132 assert(SrcRegs.size() > 1 && "Nothing to pack");
134 const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
135 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
137 LLT PackedLLT = getLLTForType(*PackedTy, DL);
139 SmallVector<LLT, 8> LLTs;
140 SmallVector<uint64_t, 8> Offsets;
141 computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
142 assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch");
144 Register Dst = MRI->createGenericVirtualRegister(PackedLLT);
145 MIRBuilder.buildUndef(Dst);
146 for (unsigned i = 0; i < SrcRegs.size(); ++i) {
147 Register NewDst = MRI->createGenericVirtualRegister(PackedLLT);
148 MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]);
155 void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg,
157 MachineIRBuilder &MIRBuilder) const {
158 assert(DstRegs.size() > 1 && "Nothing to unpack");
160 const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
162 SmallVector<LLT, 8> LLTs;
163 SmallVector<uint64_t, 8> Offsets;
164 computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
165 assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch");
167 for (unsigned i = 0; i < DstRegs.size(); ++i)
168 MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]);
171 bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder,
172 SmallVectorImpl<ArgInfo> &Args,
173 ValueHandler &Handler) const {
174 MachineFunction &MF = MIRBuilder.getMF();
175 const Function &F = MF.getFunction();
176 SmallVector<CCValAssign, 16> ArgLocs;
177 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
178 return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler);
181 bool CallLowering::handleAssignments(CCState &CCInfo,
182 SmallVectorImpl<CCValAssign> &ArgLocs,
183 MachineIRBuilder &MIRBuilder,
184 SmallVectorImpl<ArgInfo> &Args,
185 ValueHandler &Handler) const {
186 MachineFunction &MF = MIRBuilder.getMF();
187 const Function &F = MF.getFunction();
188 const DataLayout &DL = F.getParent()->getDataLayout();
190 unsigned NumArgs = Args.size();
191 for (unsigned i = 0; i != NumArgs; ++i) {
192 MVT CurVT = MVT::getVT(Args[i].Ty);
193 if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i],
194 Args[i].Flags[0], CCInfo)) {
195 if (!CurVT.isValid())
197 MVT NewVT = TLI->getRegisterTypeForCallingConv(
198 F.getContext(), F.getCallingConv(), EVT(CurVT));
200 // If we need to split the type over multiple regs, check it's a scenario
201 // we currently support.
202 unsigned NumParts = TLI->getNumRegistersForCallingConv(
203 F.getContext(), F.getCallingConv(), CurVT);
205 // For now only handle exact splits.
206 if (NewVT.getSizeInBits() * NumParts != CurVT.getSizeInBits())
210 // For incoming arguments (physregs to vregs), we could have values in
211 // physregs (or memlocs) which we want to extract and copy to vregs.
212 // During this, we might have to deal with the LLT being split across
213 // multiple regs, so we have to record this information for later.
215 // If we have outgoing args, then we have the opposite case. We have a
216 // vreg with an LLT which we want to assign to a physical location, and
217 // we might have to record that the value has to be split later.
218 if (Handler.isIncomingArgumentHandler()) {
220 // Try to use the register type if we couldn't assign the VT.
221 if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i],
222 Args[i].Flags[0], CCInfo))
225 // We're handling an incoming arg which is split over multiple regs.
226 // E.g. passing an s128 on AArch64.
227 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
228 Args[i].OrigRegs.push_back(Args[i].Regs[0]);
229 Args[i].Regs.clear();
230 Args[i].Flags.clear();
231 LLT NewLLT = getLLTForMVT(NewVT);
232 // For each split register, create and assign a vreg that will store
233 // the incoming component of the larger value. These will later be
234 // merged to form the final vreg.
235 for (unsigned Part = 0; Part < NumParts; ++Part) {
237 MIRBuilder.getMRI()->createGenericVirtualRegister(NewLLT);
238 ISD::ArgFlagsTy Flags = OrigFlags;
242 Flags.setOrigAlign(Align::None());
243 if (Part == NumParts - 1)
246 Args[i].Regs.push_back(Reg);
247 Args[i].Flags.push_back(Flags);
248 if (Handler.assignArg(i + Part, NewVT, NewVT, CCValAssign::Full,
249 Args[i], Args[i].Flags[Part], CCInfo)) {
250 // Still couldn't assign this smaller part type for some reason.
256 // Handling an outgoing arg that might need to be split.
258 return false; // Don't know how to deal with this type combination.
260 // This type is passed via multiple registers in the calling convention.
261 // We need to extract the individual parts.
262 Register LargeReg = Args[i].Regs[0];
263 LLT SmallTy = LLT::scalar(NewVT.getSizeInBits());
264 auto Unmerge = MIRBuilder.buildUnmerge(SmallTy, LargeReg);
265 assert(Unmerge->getNumOperands() == NumParts + 1);
266 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
267 // We're going to replace the regs and flags with the split ones.
268 Args[i].Regs.clear();
269 Args[i].Flags.clear();
270 for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) {
271 ISD::ArgFlagsTy Flags = OrigFlags;
275 Flags.setOrigAlign(Align::None());
276 if (PartIdx == NumParts - 1)
279 Args[i].Regs.push_back(Unmerge.getReg(PartIdx));
280 Args[i].Flags.push_back(Flags);
281 if (Handler.assignArg(i + PartIdx, NewVT, NewVT, CCValAssign::Full,
282 Args[i], Args[i].Flags[PartIdx], CCInfo))
289 for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) {
290 assert(j < ArgLocs.size() && "Skipped too many arg locs");
292 CCValAssign &VA = ArgLocs[j];
293 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
295 if (VA.needsCustom()) {
296 j += Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j));
300 // FIXME: Pack registers if we have more than one.
301 Register ArgReg = Args[i].Regs[0];
303 MVT OrigVT = MVT::getVT(Args[i].Ty);
304 MVT VAVT = VA.getValVT();
306 if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) {
307 if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
308 // Expected to be multiple regs for a single incoming arg.
309 unsigned NumArgRegs = Args[i].Regs.size();
313 assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
314 "Too many regs for number of args");
315 for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
316 // There should be Regs.size() ArgLocs per argument.
317 VA = ArgLocs[j + Part];
318 Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
321 // Merge the split registers into the expected larger result vreg
322 // of the original call.
323 MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
326 const LLT VATy(VAVT);
328 MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
329 Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
330 // If it's a vector type, we either need to truncate the elements
331 // or do an unmerge to get the lower block of elements.
332 if (VATy.isVector() &&
333 VATy.getNumElements() > OrigVT.getVectorNumElements()) {
334 const LLT OrigTy(OrigVT);
335 // Just handle the case where the VA type is 2 * original type.
336 if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
338 << "Incoming promoted vector arg has too many elts");
341 auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg});
342 MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0));
344 MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0);
346 } else if (!Handler.isIncomingArgumentHandler()) {
347 assert((j + (Args[i].Regs.size() - 1)) < ArgLocs.size() &&
348 "Too many regs for number of args");
349 // This is an outgoing argument that might have been split.
350 for (unsigned Part = 0; Part < Args[i].Regs.size(); ++Part) {
351 // There should be Regs.size() ArgLocs per argument.
352 VA = ArgLocs[j + Part];
353 Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
355 j += Args[i].Regs.size() - 1;
357 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
359 } else if (VA.isMemLoc()) {
360 // Don't currently support loading/storing a type that needs to be split
361 // to the stack. Should be easy, just not implemented yet.
362 if (Args[i].Regs.size() > 1) {
365 << "Load/store a split arg to/from the stack not implemented yet");
368 MVT VT = MVT::getVT(Args[i].Ty);
369 unsigned Size = VT == MVT::iPTR ? DL.getPointerSize()
370 : alignTo(VT.getSizeInBits(), 8) / 8;
371 unsigned Offset = VA.getLocMemOffset();
372 MachinePointerInfo MPO;
373 Register StackAddr = Handler.getStackAddress(Size, Offset, MPO);
374 Handler.assignValueToAddress(ArgReg, StackAddr, Size, MPO, VA);
376 // FIXME: Support byvals and other weirdness
383 bool CallLowering::analyzeArgInfo(CCState &CCState,
384 SmallVectorImpl<ArgInfo> &Args,
385 CCAssignFn &AssignFnFixed,
386 CCAssignFn &AssignFnVarArg) const {
387 for (unsigned i = 0, e = Args.size(); i < e; ++i) {
388 MVT VT = MVT::getVT(Args[i].Ty);
389 CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg;
390 if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) {
391 // Bail out on anything we can't handle.
392 LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString()
393 << " (arg number = " << i << "\n");
400 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
402 SmallVectorImpl<ArgInfo> &InArgs,
403 CCAssignFn &CalleeAssignFnFixed,
404 CCAssignFn &CalleeAssignFnVarArg,
405 CCAssignFn &CallerAssignFnFixed,
406 CCAssignFn &CallerAssignFnVarArg) const {
407 const Function &F = MF.getFunction();
408 CallingConv::ID CalleeCC = Info.CallConv;
409 CallingConv::ID CallerCC = F.getCallingConv();
411 if (CallerCC == CalleeCC)
414 SmallVector<CCValAssign, 16> ArgLocs1;
415 CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext());
416 if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed,
417 CalleeAssignFnVarArg))
420 SmallVector<CCValAssign, 16> ArgLocs2;
421 CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext());
422 if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed,
423 CalleeAssignFnVarArg))
426 // We need the argument locations to match up exactly. If there's more in
427 // one than the other, then we are done.
428 if (ArgLocs1.size() != ArgLocs2.size())
431 // Make sure that each location is passed in exactly the same way.
432 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
433 const CCValAssign &Loc1 = ArgLocs1[i];
434 const CCValAssign &Loc2 = ArgLocs2[i];
436 // We need both of them to be the same. So if one is a register and one
437 // isn't, we're done.
438 if (Loc1.isRegLoc() != Loc2.isRegLoc())
441 if (Loc1.isRegLoc()) {
442 // If they don't have the same register location, we're done.
443 if (Loc1.getLocReg() != Loc2.getLocReg())
446 // They matched, so we can move to the next ArgLoc.
450 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
451 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
458 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
460 LLT LocTy{VA.getLocVT()};
461 if (LocTy.getSizeInBits() == MRI.getType(ValReg).getSizeInBits())
463 switch (VA.getLocInfo()) {
465 case CCValAssign::Full:
466 case CCValAssign::BCvt:
467 // FIXME: bitconverting between vector types may or may not be a
468 // nop in big-endian situations.
470 case CCValAssign::AExt: {
471 auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
472 return MIB->getOperand(0).getReg();
474 case CCValAssign::SExt: {
475 Register NewReg = MRI.createGenericVirtualRegister(LocTy);
476 MIRBuilder.buildSExt(NewReg, ValReg);
479 case CCValAssign::ZExt: {
480 Register NewReg = MRI.createGenericVirtualRegister(LocTy);
481 MIRBuilder.buildZExt(NewReg, ValReg);
485 llvm_unreachable("unable to extend register");
488 void CallLowering::ValueHandler::anchor() {}