1 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
12 //===----------------------------------------------------------------------===//
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/Analysis/AliasAnalysis.h"
21 #include "llvm/Analysis/BranchProbabilityInfo.h"
22 #include "llvm/Analysis/ConstantFolding.h"
23 #include "llvm/Analysis/Loads.h"
24 #include "llvm/Analysis/TargetLibraryInfo.h"
25 #include "llvm/Analysis/ValueTracking.h"
26 #include "llvm/Analysis/VectorUtils.h"
27 #include "llvm/CodeGen/Analysis.h"
28 #include "llvm/CodeGen/FastISel.h"
29 #include "llvm/CodeGen/FunctionLoweringInfo.h"
30 #include "llvm/CodeGen/GCMetadata.h"
31 #include "llvm/CodeGen/GCStrategy.h"
32 #include "llvm/CodeGen/MachineFrameInfo.h"
33 #include "llvm/CodeGen/MachineFunction.h"
34 #include "llvm/CodeGen/MachineInstrBuilder.h"
35 #include "llvm/CodeGen/MachineJumpTableInfo.h"
36 #include "llvm/CodeGen/MachineModuleInfo.h"
37 #include "llvm/CodeGen/MachineRegisterInfo.h"
38 #include "llvm/CodeGen/SelectionDAG.h"
39 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
40 #include "llvm/CodeGen/StackMaps.h"
41 #include "llvm/CodeGen/WinEHFuncInfo.h"
42 #include "llvm/IR/CallingConv.h"
43 #include "llvm/IR/ConstantRange.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DataLayout.h"
46 #include "llvm/IR/DebugInfo.h"
47 #include "llvm/IR/DerivedTypes.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/GetElementPtrTypeIterator.h"
50 #include "llvm/IR/GlobalVariable.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/Instructions.h"
53 #include "llvm/IR/IntrinsicInst.h"
54 #include "llvm/IR/Intrinsics.h"
55 #include "llvm/IR/LLVMContext.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/Statepoint.h"
58 #include "llvm/MC/MCSymbol.h"
59 #include "llvm/Support/CommandLine.h"
60 #include "llvm/Support/Debug.h"
61 #include "llvm/Support/ErrorHandling.h"
62 #include "llvm/Support/MathExtras.h"
63 #include "llvm/Support/raw_ostream.h"
64 #include "llvm/Target/TargetFrameLowering.h"
65 #include "llvm/Target/TargetInstrInfo.h"
66 #include "llvm/Target/TargetIntrinsicInfo.h"
67 #include "llvm/Target/TargetLowering.h"
68 #include "llvm/Target/TargetOptions.h"
69 #include "llvm/Target/TargetSubtargetInfo.h"
74 #define DEBUG_TYPE "isel"
76 /// LimitFloatPrecision - Generate low-precision inline sequences for
77 /// some float libcalls (6, 8 or 12 bits).
78 static unsigned LimitFloatPrecision;
80 static cl::opt<unsigned, true>
81 LimitFPPrecision("limit-float-precision",
82 cl::desc("Generate low-precision inline sequences "
83 "for some float libcalls"),
84 cl::location(LimitFloatPrecision),
86 // Limit the width of DAG chains. This is important in general to prevent
87 // DAG-based analysis from blowing up. For example, alias analysis and
88 // load clustering may not complete in reasonable time. It is difficult to
89 // recognize and avoid this situation within each individual analysis, and
90 // future analyses are likely to have the same behavior. Limiting DAG width is
91 // the safe approach and will be especially important with global DAGs.
93 // MaxParallelChains default is arbitrarily high to avoid affecting
94 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
95 // sequence over this should have been converted to llvm.memcpy by the
96 // frontend. It is easy to induce this behavior with .ll code such as:
97 // %buffer = alloca [4096 x i8]
98 // %data = load [4096 x i8]* %argPtr
99 // store [4096 x i8] %data, [4096 x i8]* %buffer
100 static const unsigned MaxParallelChains = 64;
102 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
103 const SDValue *Parts, unsigned NumParts,
104 MVT PartVT, EVT ValueVT, const Value *V);
106 /// getCopyFromParts - Create a value that contains the specified legal parts
107 /// combined into the value they represent. If the parts combine to a type
108 /// larger than ValueVT then AssertOp can be used to specify whether the extra
109 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
110 /// (ISD::AssertSext).
111 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
112 const SDValue *Parts, unsigned NumParts,
113 MVT PartVT, EVT ValueVT, const Value *V,
114 Optional<ISD::NodeType> AssertOp = None) {
115 if (ValueVT.isVector())
116 return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
119 assert(NumParts > 0 && "No parts to assemble!");
120 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
121 SDValue Val = Parts[0];
124 // Assemble the value from multiple parts.
125 if (ValueVT.isInteger()) {
126 unsigned PartBits = PartVT.getSizeInBits();
127 unsigned ValueBits = ValueVT.getSizeInBits();
129 // Assemble the power of 2 part.
130 unsigned RoundParts = NumParts & (NumParts - 1) ?
131 1 << Log2_32(NumParts) : NumParts;
132 unsigned RoundBits = PartBits * RoundParts;
133 EVT RoundVT = RoundBits == ValueBits ?
134 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
137 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
139 if (RoundParts > 2) {
140 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
142 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
143 RoundParts / 2, PartVT, HalfVT, V);
145 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
146 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
149 if (DAG.getDataLayout().isBigEndian())
152 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
154 if (RoundParts < NumParts) {
155 // Assemble the trailing non-power-of-2 part.
156 unsigned OddParts = NumParts - RoundParts;
157 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
158 Hi = getCopyFromParts(DAG, DL,
159 Parts + RoundParts, OddParts, PartVT, OddVT, V);
161 // Combine the round and odd parts.
163 if (DAG.getDataLayout().isBigEndian())
165 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
166 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
168 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
169 DAG.getConstant(Lo.getValueSizeInBits(), DL,
170 TLI.getPointerTy(DAG.getDataLayout())));
171 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
172 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
174 } else if (PartVT.isFloatingPoint()) {
175 // FP split into multiple FP parts (for ppcf128)
176 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
179 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
180 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
181 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
183 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
185 // FP split into integer parts (soft fp)
186 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
187 !PartVT.isVector() && "Unexpected split");
188 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
189 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
193 // There is now one part, held in Val. Correct it to match ValueVT.
194 // PartEVT is the type of the register class that holds the value.
195 // ValueVT is the type of the inline asm operation.
196 EVT PartEVT = Val.getValueType();
198 if (PartEVT == ValueVT)
201 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
202 ValueVT.bitsLT(PartEVT)) {
203 // For an FP value in an integer part, we need to truncate to the right
205 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
206 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
209 // Handle types that have the same size.
210 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
211 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
213 // Handle types with different sizes.
214 if (PartEVT.isInteger() && ValueVT.isInteger()) {
215 if (ValueVT.bitsLT(PartEVT)) {
216 // For a truncate, see if we have any information to
217 // indicate whether the truncated bits will always be
218 // zero or sign-extension.
219 if (AssertOp.hasValue())
220 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
221 DAG.getValueType(ValueVT));
222 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
224 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
227 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
228 // FP_ROUND's are always exact here.
229 if (ValueVT.bitsLT(Val.getValueType()))
231 ISD::FP_ROUND, DL, ValueVT, Val,
232 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
234 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
237 llvm_unreachable("Unknown mismatch!");
240 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
241 const Twine &ErrMsg) {
242 const Instruction *I = dyn_cast_or_null<Instruction>(V);
244 return Ctx.emitError(ErrMsg);
246 const char *AsmError = ", possible invalid constraint for vector type";
247 if (const CallInst *CI = dyn_cast<CallInst>(I))
248 if (isa<InlineAsm>(CI->getCalledValue()))
249 return Ctx.emitError(I, ErrMsg + AsmError);
251 return Ctx.emitError(I, ErrMsg);
254 /// getCopyFromPartsVector - Create a value that contains the specified legal
255 /// parts combined into the value they represent. If the parts combine to a
256 /// type larger than ValueVT then AssertOp can be used to specify whether the
257 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
258 /// ValueVT (ISD::AssertSext).
259 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
260 const SDValue *Parts, unsigned NumParts,
261 MVT PartVT, EVT ValueVT, const Value *V) {
262 assert(ValueVT.isVector() && "Not a vector value");
263 assert(NumParts > 0 && "No parts to assemble!");
264 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
265 SDValue Val = Parts[0];
267 // Handle a multi-element vector.
271 unsigned NumIntermediates;
273 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
274 NumIntermediates, RegisterVT);
275 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
276 NumParts = NumRegs; // Silence a compiler warning.
277 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
278 assert(RegisterVT.getSizeInBits() ==
279 Parts[0].getSimpleValueType().getSizeInBits() &&
280 "Part type sizes don't match!");
282 // Assemble the parts into intermediate operands.
283 SmallVector<SDValue, 8> Ops(NumIntermediates);
284 if (NumIntermediates == NumParts) {
285 // If the register was not expanded, truncate or copy the value,
287 for (unsigned i = 0; i != NumParts; ++i)
288 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
289 PartVT, IntermediateVT, V);
290 } else if (NumParts > 0) {
291 // If the intermediate type was expanded, build the intermediate
292 // operands from the parts.
293 assert(NumParts % NumIntermediates == 0 &&
294 "Must expand into a divisible number of parts!");
295 unsigned Factor = NumParts / NumIntermediates;
296 for (unsigned i = 0; i != NumIntermediates; ++i)
297 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
298 PartVT, IntermediateVT, V);
301 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
302 // intermediate operands.
303 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
308 // There is now one part, held in Val. Correct it to match ValueVT.
309 EVT PartEVT = Val.getValueType();
311 if (PartEVT == ValueVT)
314 if (PartEVT.isVector()) {
315 // If the element type of the source/dest vectors are the same, but the
316 // parts vector has more elements than the value vector, then we have a
317 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
319 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
320 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
321 "Cannot narrow, it would be a lossy transformation");
323 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
324 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
327 // Vector/Vector bitcast.
328 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
329 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
331 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
332 "Cannot handle this kind of promotion");
333 // Promoted vector extract
334 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
338 // Trivial bitcast if the types are the same size and the destination
339 // vector type is legal.
340 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
341 TLI.isTypeLegal(ValueVT))
342 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
344 // Handle cases such as i8 -> <1 x i1>
345 if (ValueVT.getVectorNumElements() != 1) {
346 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
347 "non-trivial scalar-to-vector conversion");
348 return DAG.getUNDEF(ValueVT);
351 EVT ValueSVT = ValueVT.getVectorElementType();
352 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
353 Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
354 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
356 return DAG.getBuildVector(ValueVT, DL, Val);
359 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
360 SDValue Val, SDValue *Parts, unsigned NumParts,
361 MVT PartVT, const Value *V);
363 /// getCopyToParts - Create a series of nodes that contain the specified value
364 /// split into legal parts. If the parts contain more bits than Val, then, for
365 /// integers, ExtendKind can be used to specify how to generate the extra bits.
366 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
367 SDValue *Parts, unsigned NumParts, MVT PartVT,
369 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
370 EVT ValueVT = Val.getValueType();
372 // Handle the vector case separately.
373 if (ValueVT.isVector())
374 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
376 unsigned PartBits = PartVT.getSizeInBits();
377 unsigned OrigNumParts = NumParts;
378 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
379 "Copying to an illegal type!");
384 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
385 EVT PartEVT = PartVT;
386 if (PartEVT == ValueVT) {
387 assert(NumParts == 1 && "No-op copy with multiple parts!");
392 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
393 // If the parts cover more bits than the value has, promote the value.
394 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
395 assert(NumParts == 1 && "Do not know what to promote to!");
396 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
398 if (ValueVT.isFloatingPoint()) {
399 // FP values need to be bitcast, then extended if they are being put
400 // into a larger container.
401 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
402 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
404 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
405 ValueVT.isInteger() &&
406 "Unknown mismatch!");
407 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
408 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
409 if (PartVT == MVT::x86mmx)
410 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
412 } else if (PartBits == ValueVT.getSizeInBits()) {
413 // Different types of the same size.
414 assert(NumParts == 1 && PartEVT != ValueVT);
415 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
416 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
417 // If the parts cover less bits than value has, truncate the value.
418 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
419 ValueVT.isInteger() &&
420 "Unknown mismatch!");
421 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
422 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
423 if (PartVT == MVT::x86mmx)
424 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
427 // The value may have changed - recompute ValueVT.
428 ValueVT = Val.getValueType();
429 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
430 "Failed to tile the value with PartVT!");
433 if (PartEVT != ValueVT) {
434 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
435 "scalar-to-vector conversion failed");
436 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
443 // Expand the value into multiple parts.
444 if (NumParts & (NumParts - 1)) {
445 // The number of parts is not a power of 2. Split off and copy the tail.
446 assert(PartVT.isInteger() && ValueVT.isInteger() &&
447 "Do not know what to expand to!");
448 unsigned RoundParts = 1 << Log2_32(NumParts);
449 unsigned RoundBits = RoundParts * PartBits;
450 unsigned OddParts = NumParts - RoundParts;
451 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
452 DAG.getIntPtrConstant(RoundBits, DL));
453 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
455 if (DAG.getDataLayout().isBigEndian())
456 // The odd parts were reversed by getCopyToParts - unreverse them.
457 std::reverse(Parts + RoundParts, Parts + NumParts);
459 NumParts = RoundParts;
460 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
461 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
464 // The number of parts is a power of 2. Repeatedly bisect the value using
466 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
467 EVT::getIntegerVT(*DAG.getContext(),
468 ValueVT.getSizeInBits()),
471 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
472 for (unsigned i = 0; i < NumParts; i += StepSize) {
473 unsigned ThisBits = StepSize * PartBits / 2;
474 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
475 SDValue &Part0 = Parts[i];
476 SDValue &Part1 = Parts[i+StepSize/2];
478 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
479 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
480 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
481 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
483 if (ThisBits == PartBits && ThisVT != PartVT) {
484 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
485 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
490 if (DAG.getDataLayout().isBigEndian())
491 std::reverse(Parts, Parts + OrigNumParts);
495 /// getCopyToPartsVector - Create a series of nodes that contain the specified
496 /// value split into legal parts.
497 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
498 SDValue Val, SDValue *Parts, unsigned NumParts,
499 MVT PartVT, const Value *V) {
500 EVT ValueVT = Val.getValueType();
501 assert(ValueVT.isVector() && "Not a vector");
502 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
505 EVT PartEVT = PartVT;
506 if (PartEVT == ValueVT) {
508 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
509 // Bitconvert vector->vector case.
510 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
511 } else if (PartVT.isVector() &&
512 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
513 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
514 EVT ElementVT = PartVT.getVectorElementType();
515 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
517 SmallVector<SDValue, 16> Ops;
518 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
519 Ops.push_back(DAG.getNode(
520 ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
521 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
523 for (unsigned i = ValueVT.getVectorNumElements(),
524 e = PartVT.getVectorNumElements(); i != e; ++i)
525 Ops.push_back(DAG.getUNDEF(ElementVT));
527 Val = DAG.getBuildVector(PartVT, DL, Ops);
529 // FIXME: Use CONCAT for 2x -> 4x.
531 //SDValue UndefElts = DAG.getUNDEF(VectorTy);
532 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
533 } else if (PartVT.isVector() &&
534 PartEVT.getVectorElementType().bitsGE(
535 ValueVT.getVectorElementType()) &&
536 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
538 // Promoted vector extract
539 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
541 // Vector -> scalar conversion.
542 assert(ValueVT.getVectorNumElements() == 1 &&
543 "Only trivial vector-to-scalar conversions should get here!");
545 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
546 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
549 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
554 // Handle a multi-element vector.
557 unsigned NumIntermediates;
558 unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
560 NumIntermediates, RegisterVT);
561 unsigned NumElements = ValueVT.getVectorNumElements();
563 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
564 NumParts = NumRegs; // Silence a compiler warning.
565 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
567 // Split the vector into intermediate operands.
568 SmallVector<SDValue, 8> Ops(NumIntermediates);
569 for (unsigned i = 0; i != NumIntermediates; ++i) {
570 if (IntermediateVT.isVector())
572 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
573 DAG.getConstant(i * (NumElements / NumIntermediates), DL,
574 TLI.getVectorIdxTy(DAG.getDataLayout())));
576 Ops[i] = DAG.getNode(
577 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
578 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
581 // Split the intermediate operands into legal parts.
582 if (NumParts == NumIntermediates) {
583 // If the register was not expanded, promote or copy the value,
585 for (unsigned i = 0; i != NumParts; ++i)
586 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
587 } else if (NumParts > 0) {
588 // If the intermediate type was expanded, split each the value into
590 assert(NumIntermediates != 0 && "division by zero");
591 assert(NumParts % NumIntermediates == 0 &&
592 "Must expand into a divisible number of parts!");
593 unsigned Factor = NumParts / NumIntermediates;
594 for (unsigned i = 0; i != NumIntermediates; ++i)
595 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
599 RegsForValue::RegsForValue() {}
601 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt,
603 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
605 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
606 const DataLayout &DL, unsigned Reg, Type *Ty) {
607 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
609 for (EVT ValueVT : ValueVTs) {
610 unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
611 MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
612 for (unsigned i = 0; i != NumRegs; ++i)
613 Regs.push_back(Reg + i);
614 RegVTs.push_back(RegisterVT);
619 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
620 FunctionLoweringInfo &FuncInfo,
621 const SDLoc &dl, SDValue &Chain,
622 SDValue *Flag, const Value *V) const {
623 // A Value with type {} or [0 x %t] needs no registers.
624 if (ValueVTs.empty())
627 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
629 // Assemble the legal parts into the final values.
630 SmallVector<SDValue, 4> Values(ValueVTs.size());
631 SmallVector<SDValue, 8> Parts;
632 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
633 // Copy the legal parts from the registers.
634 EVT ValueVT = ValueVTs[Value];
635 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
636 MVT RegisterVT = RegVTs[Value];
638 Parts.resize(NumRegs);
639 for (unsigned i = 0; i != NumRegs; ++i) {
642 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
644 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
645 *Flag = P.getValue(2);
648 Chain = P.getValue(1);
651 // If the source register was virtual and if we know something about it,
652 // add an assert node.
653 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
654 !RegisterVT.isInteger() || RegisterVT.isVector())
657 const FunctionLoweringInfo::LiveOutInfo *LOI =
658 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
662 unsigned RegSize = RegisterVT.getSizeInBits();
663 unsigned NumSignBits = LOI->NumSignBits;
664 unsigned NumZeroBits = LOI->Known.Zero.countLeadingOnes();
666 if (NumZeroBits == RegSize) {
667 // The current value is a zero.
668 // Explicitly express that as it would be easier for
669 // optimizations to kick in.
670 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
674 // FIXME: We capture more information than the dag can represent. For
675 // now, just use the tightest assertzext/assertsext possible.
677 EVT FromVT(MVT::Other);
678 if (NumSignBits == RegSize) {
679 isSExt = true; // ASSERT SEXT 1
681 } else if (NumZeroBits >= RegSize - 1) {
682 isSExt = false; // ASSERT ZEXT 1
684 } else if (NumSignBits > RegSize - 8) {
685 isSExt = true; // ASSERT SEXT 8
687 } else if (NumZeroBits >= RegSize - 8) {
688 isSExt = false; // ASSERT ZEXT 8
690 } else if (NumSignBits > RegSize - 16) {
691 isSExt = true; // ASSERT SEXT 16
693 } else if (NumZeroBits >= RegSize - 16) {
694 isSExt = false; // ASSERT ZEXT 16
696 } else if (NumSignBits > RegSize - 32) {
697 isSExt = true; // ASSERT SEXT 32
699 } else if (NumZeroBits >= RegSize - 32) {
700 isSExt = false; // ASSERT ZEXT 32
705 // Add an assertion node.
706 assert(FromVT != MVT::Other);
707 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
708 RegisterVT, P, DAG.getValueType(FromVT));
711 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
712 NumRegs, RegisterVT, ValueVT, V);
717 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
720 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
721 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
723 ISD::NodeType PreferredExtendType) const {
724 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
725 ISD::NodeType ExtendKind = PreferredExtendType;
727 // Get the list of the values's legal parts.
728 unsigned NumRegs = Regs.size();
729 SmallVector<SDValue, 8> Parts(NumRegs);
730 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
731 EVT ValueVT = ValueVTs[Value];
732 unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
733 MVT RegisterVT = RegVTs[Value];
735 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
736 ExtendKind = ISD::ZERO_EXTEND;
738 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
739 &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
743 // Copy the parts into the registers.
744 SmallVector<SDValue, 8> Chains(NumRegs);
745 for (unsigned i = 0; i != NumRegs; ++i) {
748 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
750 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
751 *Flag = Part.getValue(1);
754 Chains[i] = Part.getValue(0);
757 if (NumRegs == 1 || Flag)
758 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
759 // flagged to it. That is the CopyToReg nodes and the user are considered
760 // a single scheduling unit. If we create a TokenFactor and return it as
761 // chain, then the TokenFactor is both a predecessor (operand) of the
762 // user as well as a successor (the TF operands are flagged to the user).
763 // c1, f1 = CopyToReg
764 // c2, f2 = CopyToReg
765 // c3 = TokenFactor c1, c2
768 Chain = Chains[NumRegs-1];
770 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
773 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
774 unsigned MatchingIdx, const SDLoc &dl,
776 std::vector<SDValue> &Ops) const {
777 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
779 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
781 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
782 else if (!Regs.empty() &&
783 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
784 // Put the register class of the virtual registers in the flag word. That
785 // way, later passes can recompute register class constraints for inline
786 // assembly as well as normal instructions.
787 // Don't do this for tied operands that can use the regclass information
789 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
790 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
791 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
794 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
797 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
798 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
799 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
800 MVT RegisterVT = RegVTs[Value];
801 for (unsigned i = 0; i != NumRegs; ++i) {
802 assert(Reg < Regs.size() && "Mismatch in # registers expected");
803 unsigned TheReg = Regs[Reg++];
804 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
806 if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
807 // If we clobbered the stack pointer, MFI should know about it.
808 assert(DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment());
814 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
815 const TargetLibraryInfo *li) {
819 DL = &DAG.getDataLayout();
820 Context = DAG.getContext();
821 LPadToCallSiteMap.clear();
824 void SelectionDAGBuilder::clear() {
826 UnusedArgNodeMap.clear();
827 PendingLoads.clear();
828 PendingExports.clear();
831 SDNodeOrder = LowestSDNodeOrder;
832 StatepointLowering.clear();
835 void SelectionDAGBuilder::clearDanglingDebugInfo() {
836 DanglingDebugInfoMap.clear();
839 SDValue SelectionDAGBuilder::getRoot() {
840 if (PendingLoads.empty())
841 return DAG.getRoot();
843 if (PendingLoads.size() == 1) {
844 SDValue Root = PendingLoads[0];
846 PendingLoads.clear();
850 // Otherwise, we have to make a token factor node.
851 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
853 PendingLoads.clear();
858 SDValue SelectionDAGBuilder::getControlRoot() {
859 SDValue Root = DAG.getRoot();
861 if (PendingExports.empty())
864 // Turn all of the CopyToReg chains into one factored node.
865 if (Root.getOpcode() != ISD::EntryToken) {
866 unsigned i = 0, e = PendingExports.size();
867 for (; i != e; ++i) {
868 assert(PendingExports[i].getNode()->getNumOperands() > 1);
869 if (PendingExports[i].getNode()->getOperand(0) == Root)
870 break; // Don't add the root if we already indirectly depend on it.
874 PendingExports.push_back(Root);
877 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
879 PendingExports.clear();
884 void SelectionDAGBuilder::visit(const Instruction &I) {
885 // Set up outgoing PHI node register values before emitting the terminator.
886 if (isa<TerminatorInst>(&I)) {
887 HandlePHINodesInSuccessorBlocks(I.getParent());
890 // Increase the SDNodeOrder if dealing with a non-debug instruction.
891 if (!isa<DbgInfoIntrinsic>(I))
896 visit(I.getOpcode(), I);
898 if (!isa<TerminatorInst>(&I) && !HasTailCall &&
899 !isStatepoint(&I)) // statepoints handle their exports internally
900 CopyToExportRegsIfNeeded(&I);
905 void SelectionDAGBuilder::visitPHI(const PHINode &) {
906 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
909 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
910 // Note: this doesn't use InstVisitor, because it has to work with
911 // ConstantExpr's in addition to instructions.
913 default: llvm_unreachable("Unknown instruction type encountered!");
914 // Build the switch statement using the Instruction.def file.
915 #define HANDLE_INST(NUM, OPCODE, CLASS) \
916 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
917 #include "llvm/IR/Instruction.def"
921 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
922 // generate the debug data structures now that we've seen its definition.
923 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
925 DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
927 const DbgValueInst *DI = DDI.getDI();
928 DebugLoc dl = DDI.getdl();
929 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
930 DILocalVariable *Variable = DI->getVariable();
931 DIExpression *Expr = DI->getExpression();
932 assert(Variable->isValidLocationForIntrinsic(dl) &&
933 "Expected inlined-at fields to agree");
934 uint64_t Offset = DI->getOffset();
937 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
939 SDV = getDbgValue(Val, Variable, Expr, Offset, dl, DbgSDNodeOrder);
940 DAG.AddDbgValue(SDV, Val.getNode(), false);
943 DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
944 DanglingDebugInfoMap[V] = DanglingDebugInfo();
948 /// getCopyFromRegs - If there was virtual register allocated for the value V
949 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
950 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
951 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
954 if (It != FuncInfo.ValueMap.end()) {
955 unsigned InReg = It->second;
956 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
957 DAG.getDataLayout(), InReg, Ty);
958 SDValue Chain = DAG.getEntryNode();
959 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
960 resolveDanglingDebugInfo(V, Result);
966 /// getValue - Return an SDValue for the given Value.
967 SDValue SelectionDAGBuilder::getValue(const Value *V) {
968 // If we already have an SDValue for this value, use it. It's important
969 // to do this first, so that we don't create a CopyFromReg if we already
970 // have a regular SDValue.
971 SDValue &N = NodeMap[V];
972 if (N.getNode()) return N;
974 // If there's a virtual register allocated and initialized for this
976 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
979 // Otherwise create a new SDValue and remember it.
980 SDValue Val = getValueImpl(V);
982 resolveDanglingDebugInfo(V, Val);
986 // Return true if SDValue exists for the given Value
987 bool SelectionDAGBuilder::findValue(const Value *V) const {
988 return (NodeMap.find(V) != NodeMap.end()) ||
989 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
992 /// getNonRegisterValue - Return an SDValue for the given Value, but
993 /// don't look in FuncInfo.ValueMap for a virtual register.
994 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
995 // If we already have an SDValue for this value, use it.
996 SDValue &N = NodeMap[V];
998 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
999 // Remove the debug location from the node as the node is about to be used
1000 // in a location which may differ from the original debug location. This
1001 // is relevant to Constant and ConstantFP nodes because they can appear
1002 // as constant expressions inside PHI nodes.
1003 N->setDebugLoc(DebugLoc());
1008 // Otherwise create a new SDValue and remember it.
1009 SDValue Val = getValueImpl(V);
1011 resolveDanglingDebugInfo(V, Val);
1015 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1016 /// Create an SDValue for the given value.
1017 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1018 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1020 if (const Constant *C = dyn_cast<Constant>(V)) {
1021 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1023 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1024 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1026 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1027 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1029 if (isa<ConstantPointerNull>(C)) {
1030 unsigned AS = V->getType()->getPointerAddressSpace();
1031 return DAG.getConstant(0, getCurSDLoc(),
1032 TLI.getPointerTy(DAG.getDataLayout(), AS));
1035 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1036 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1038 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1039 return DAG.getUNDEF(VT);
1041 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1042 visit(CE->getOpcode(), *CE);
1043 SDValue N1 = NodeMap[V];
1044 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1048 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1049 SmallVector<SDValue, 4> Constants;
1050 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1052 SDNode *Val = getValue(*OI).getNode();
1053 // If the operand is an empty aggregate, there are no values.
1055 // Add each leaf value from the operand to the Constants list
1056 // to form a flattened list of all the values.
1057 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1058 Constants.push_back(SDValue(Val, i));
1061 return DAG.getMergeValues(Constants, getCurSDLoc());
1064 if (const ConstantDataSequential *CDS =
1065 dyn_cast<ConstantDataSequential>(C)) {
1066 SmallVector<SDValue, 4> Ops;
1067 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1068 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1069 // Add each leaf value from the operand to the Constants list
1070 // to form a flattened list of all the values.
1071 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1072 Ops.push_back(SDValue(Val, i));
1075 if (isa<ArrayType>(CDS->getType()))
1076 return DAG.getMergeValues(Ops, getCurSDLoc());
1077 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1080 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1081 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1082 "Unknown struct or array constant!");
1084 SmallVector<EVT, 4> ValueVTs;
1085 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1086 unsigned NumElts = ValueVTs.size();
1088 return SDValue(); // empty struct
1089 SmallVector<SDValue, 4> Constants(NumElts);
1090 for (unsigned i = 0; i != NumElts; ++i) {
1091 EVT EltVT = ValueVTs[i];
1092 if (isa<UndefValue>(C))
1093 Constants[i] = DAG.getUNDEF(EltVT);
1094 else if (EltVT.isFloatingPoint())
1095 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1097 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1100 return DAG.getMergeValues(Constants, getCurSDLoc());
1103 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1104 return DAG.getBlockAddress(BA, VT);
1106 VectorType *VecTy = cast<VectorType>(V->getType());
1107 unsigned NumElements = VecTy->getNumElements();
1109 // Now that we know the number and type of the elements, get that number of
1110 // elements into the Ops array based on what kind of constant it is.
1111 SmallVector<SDValue, 16> Ops;
1112 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1113 for (unsigned i = 0; i != NumElements; ++i)
1114 Ops.push_back(getValue(CV->getOperand(i)));
1116 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1118 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1121 if (EltVT.isFloatingPoint())
1122 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1124 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1125 Ops.assign(NumElements, Op);
1128 // Create a BUILD_VECTOR node.
1129 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1132 // If this is a static alloca, generate it as the frameindex instead of
1134 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1135 DenseMap<const AllocaInst*, int>::iterator SI =
1136 FuncInfo.StaticAllocaMap.find(AI);
1137 if (SI != FuncInfo.StaticAllocaMap.end())
1138 return DAG.getFrameIndex(SI->second,
1139 TLI.getFrameIndexTy(DAG.getDataLayout()));
1142 // If this is an instruction which fast-isel has deferred, select it now.
1143 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1144 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1145 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1147 SDValue Chain = DAG.getEntryNode();
1148 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1151 llvm_unreachable("Can't get register for value!");
1154 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1155 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1156 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1157 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1158 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1159 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1160 if (IsMSVCCXX || IsCoreCLR)
1161 CatchPadMBB->setIsEHFuncletEntry();
1163 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot()));
1166 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1167 // Update machine-CFG edge.
1168 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1169 FuncInfo.MBB->addSuccessor(TargetMBB);
1171 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1172 bool IsSEH = isAsynchronousEHPersonality(Pers);
1174 // If this is not a fall-through branch or optimizations are switched off,
1176 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1177 TM.getOptLevel() == CodeGenOpt::None)
1178 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1179 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1183 // Figure out the funclet membership for the catchret's successor.
1184 // This will be used by the FuncletLayout pass to determine how to order the
1186 // A 'catchret' returns to the outer scope's color.
1187 Value *ParentPad = I.getCatchSwitchParentPad();
1188 const BasicBlock *SuccessorColor;
1189 if (isa<ConstantTokenNone>(ParentPad))
1190 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1192 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1193 assert(SuccessorColor && "No parent funclet for catchret!");
1194 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1195 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1197 // Create the terminator node.
1198 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1199 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1200 DAG.getBasicBlock(SuccessorColorMBB));
1204 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1205 // Don't emit any special code for the cleanuppad instruction. It just marks
1206 // the start of a funclet.
1207 FuncInfo.MBB->setIsEHFuncletEntry();
1208 FuncInfo.MBB->setIsCleanupFuncletEntry();
1211 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1212 /// many places it could ultimately go. In the IR, we have a single unwind
1213 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1214 /// This function skips over imaginary basic blocks that hold catchswitch
1215 /// instructions, and finds all the "real" machine
1216 /// basic block destinations. As those destinations may not be successors of
1217 /// EHPadBB, here we also calculate the edge probability to those destinations.
1218 /// The passed-in Prob is the edge probability to EHPadBB.
1219 static void findUnwindDestinations(
1220 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1221 BranchProbability Prob,
1222 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1224 EHPersonality Personality =
1225 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1226 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1227 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1230 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1231 BasicBlock *NewEHPadBB = nullptr;
1232 if (isa<LandingPadInst>(Pad)) {
1233 // Stop on landingpads. They are not funclets.
1234 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1236 } else if (isa<CleanupPadInst>(Pad)) {
1237 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1239 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1240 UnwindDests.back().first->setIsEHFuncletEntry();
1242 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1243 // Add the catchpad handlers to the possible destinations.
1244 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1245 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1246 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1247 if (IsMSVCCXX || IsCoreCLR)
1248 UnwindDests.back().first->setIsEHFuncletEntry();
1250 NewEHPadBB = CatchSwitch->getUnwindDest();
1255 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1256 if (BPI && NewEHPadBB)
1257 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1258 EHPadBB = NewEHPadBB;
1262 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1263 // Update successor info.
1264 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1265 auto UnwindDest = I.getUnwindDest();
1266 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1267 BranchProbability UnwindDestProb =
1269 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1270 : BranchProbability::getZero();
1271 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1272 for (auto &UnwindDest : UnwindDests) {
1273 UnwindDest.first->setIsEHPad();
1274 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1276 FuncInfo.MBB->normalizeSuccProbs();
1278 // Create the terminator node.
1280 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1284 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1285 report_fatal_error("visitCatchSwitch not yet implemented!");
1288 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1289 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1290 auto &DL = DAG.getDataLayout();
1291 SDValue Chain = getControlRoot();
1292 SmallVector<ISD::OutputArg, 8> Outs;
1293 SmallVector<SDValue, 8> OutVals;
1295 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1298 // %val = call <ty> @llvm.experimental.deoptimize()
1302 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1303 LowerDeoptimizingReturn();
1307 if (!FuncInfo.CanLowerReturn) {
1308 unsigned DemoteReg = FuncInfo.DemoteRegister;
1309 const Function *F = I.getParent()->getParent();
1311 // Emit a store of the return value through the virtual register.
1312 // Leave Outs empty so that LowerReturn won't try to load return
1313 // registers the usual way.
1314 SmallVector<EVT, 1> PtrValueVTs;
1315 ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()),
1318 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1319 DemoteReg, PtrValueVTs[0]);
1320 SDValue RetOp = getValue(I.getOperand(0));
1322 SmallVector<EVT, 4> ValueVTs;
1323 SmallVector<uint64_t, 4> Offsets;
1324 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1325 unsigned NumValues = ValueVTs.size();
1327 // An aggregate return value cannot wrap around the address space, so
1328 // offsets to its parts don't wrap either.
1330 Flags.setNoUnsignedWrap(true);
1332 SmallVector<SDValue, 4> Chains(NumValues);
1333 for (unsigned i = 0; i != NumValues; ++i) {
1334 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
1335 RetPtr.getValueType(), RetPtr,
1336 DAG.getIntPtrConstant(Offsets[i],
1339 Chains[i] = DAG.getStore(Chain, getCurSDLoc(),
1340 SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1341 // FIXME: better loc info would be nice.
1342 Add, MachinePointerInfo());
1345 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1346 MVT::Other, Chains);
1347 } else if (I.getNumOperands() != 0) {
1348 SmallVector<EVT, 4> ValueVTs;
1349 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1350 unsigned NumValues = ValueVTs.size();
1352 SDValue RetOp = getValue(I.getOperand(0));
1354 const Function *F = I.getParent()->getParent();
1356 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1357 if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1359 ExtendKind = ISD::SIGN_EXTEND;
1360 else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1362 ExtendKind = ISD::ZERO_EXTEND;
1364 LLVMContext &Context = F->getContext();
1365 bool RetInReg = F->getAttributes().hasAttribute(
1366 AttributeList::ReturnIndex, Attribute::InReg);
1368 for (unsigned j = 0; j != NumValues; ++j) {
1369 EVT VT = ValueVTs[j];
1371 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1372 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1374 unsigned NumParts = TLI.getNumRegisters(Context, VT);
1375 MVT PartVT = TLI.getRegisterType(Context, VT);
1376 SmallVector<SDValue, 4> Parts(NumParts);
1377 getCopyToParts(DAG, getCurSDLoc(),
1378 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1379 &Parts[0], NumParts, PartVT, &I, ExtendKind);
1381 // 'inreg' on function refers to return value
1382 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1386 // Propagate extension type if any
1387 if (ExtendKind == ISD::SIGN_EXTEND)
1389 else if (ExtendKind == ISD::ZERO_EXTEND)
1392 for (unsigned i = 0; i < NumParts; ++i) {
1393 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1394 VT, /*isfixed=*/true, 0, 0));
1395 OutVals.push_back(Parts[i]);
1401 // Push in swifterror virtual register as the last element of Outs. This makes
1402 // sure swifterror virtual register will be returned in the swifterror
1403 // physical register.
1404 const Function *F = I.getParent()->getParent();
1405 if (TLI.supportSwiftError() &&
1406 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1407 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1408 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1409 Flags.setSwiftError();
1410 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1411 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1412 true /*isfixed*/, 1 /*origidx*/,
1414 // Create SDNode for the swifterror virtual register.
1415 OutVals.push_back(DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVReg(
1416 FuncInfo.MBB, FuncInfo.SwiftErrorArg),
1417 EVT(TLI.getPointerTy(DL))));
1420 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1421 CallingConv::ID CallConv =
1422 DAG.getMachineFunction().getFunction()->getCallingConv();
1423 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1424 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1426 // Verify that the target's LowerReturn behaved as expected.
1427 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1428 "LowerReturn didn't return a valid chain!");
1430 // Update the DAG with the new chain value resulting from return lowering.
1434 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1435 /// created for it, emit nodes to copy the value into the virtual
1437 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1439 if (V->getType()->isEmptyTy())
1442 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1443 if (VMI != FuncInfo.ValueMap.end()) {
1444 assert(!V->use_empty() && "Unused value assigned virtual registers!");
1445 CopyValueToVirtualRegister(V, VMI->second);
1449 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1450 /// the current basic block, add it to ValueMap now so that we'll get a
1452 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1453 // No need to export constants.
1454 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1456 // Already exported?
1457 if (FuncInfo.isExportedInst(V)) return;
1459 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1460 CopyValueToVirtualRegister(V, Reg);
1463 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1464 const BasicBlock *FromBB) {
1465 // The operands of the setcc have to be in this block. We don't know
1466 // how to export them from some other block.
1467 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1468 // Can export from current BB.
1469 if (VI->getParent() == FromBB)
1472 // Is already exported, noop.
1473 return FuncInfo.isExportedInst(V);
1476 // If this is an argument, we can export it if the BB is the entry block or
1477 // if it is already exported.
1478 if (isa<Argument>(V)) {
1479 if (FromBB == &FromBB->getParent()->getEntryBlock())
1482 // Otherwise, can only export this if it is already exported.
1483 return FuncInfo.isExportedInst(V);
1486 // Otherwise, constants can always be exported.
1490 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1492 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1493 const MachineBasicBlock *Dst) const {
1494 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1495 const BasicBlock *SrcBB = Src->getBasicBlock();
1496 const BasicBlock *DstBB = Dst->getBasicBlock();
1498 // If BPI is not available, set the default probability as 1 / N, where N is
1499 // the number of successors.
1500 auto SuccSize = std::max<uint32_t>(
1501 std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
1502 return BranchProbability(1, SuccSize);
1504 return BPI->getEdgeProbability(SrcBB, DstBB);
1507 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1508 MachineBasicBlock *Dst,
1509 BranchProbability Prob) {
1511 Src->addSuccessorWithoutProb(Dst);
1513 if (Prob.isUnknown())
1514 Prob = getEdgeProbability(Src, Dst);
1515 Src->addSuccessor(Dst, Prob);
1519 static bool InBlock(const Value *V, const BasicBlock *BB) {
1520 if (const Instruction *I = dyn_cast<Instruction>(V))
1521 return I->getParent() == BB;
1525 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1526 /// This function emits a branch and is used at the leaves of an OR or an
1527 /// AND operator tree.
1530 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1531 MachineBasicBlock *TBB,
1532 MachineBasicBlock *FBB,
1533 MachineBasicBlock *CurBB,
1534 MachineBasicBlock *SwitchBB,
1535 BranchProbability TProb,
1536 BranchProbability FProb,
1538 const BasicBlock *BB = CurBB->getBasicBlock();
1540 // If the leaf of the tree is a comparison, merge the condition into
1542 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1543 // The operands of the cmp have to be in this block. We don't know
1544 // how to export them from some other block. If this is the first block
1545 // of the sequence, no exporting is needed.
1546 if (CurBB == SwitchBB ||
1547 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1548 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1549 ISD::CondCode Condition;
1550 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1551 ICmpInst::Predicate Pred =
1552 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1553 Condition = getICmpCondCode(Pred);
1555 const FCmpInst *FC = cast<FCmpInst>(Cond);
1556 FCmpInst::Predicate Pred =
1557 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1558 Condition = getFCmpCondCode(Pred);
1559 if (TM.Options.NoNaNsFPMath)
1560 Condition = getFCmpCodeWithoutNaN(Condition);
1563 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1564 TBB, FBB, CurBB, TProb, FProb);
1565 SwitchCases.push_back(CB);
1570 // Create a CaseBlock record representing this branch.
1571 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1572 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1573 nullptr, TBB, FBB, CurBB, TProb, FProb);
1574 SwitchCases.push_back(CB);
1577 /// FindMergedConditions - If Cond is an expression like
1578 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1579 MachineBasicBlock *TBB,
1580 MachineBasicBlock *FBB,
1581 MachineBasicBlock *CurBB,
1582 MachineBasicBlock *SwitchBB,
1583 Instruction::BinaryOps Opc,
1584 BranchProbability TProb,
1585 BranchProbability FProb,
1587 // Skip over not part of the tree and remember to invert op and operands at
1589 if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1590 const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1591 if (InBlock(CondOp, CurBB->getBasicBlock())) {
1592 FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1598 const Instruction *BOp = dyn_cast<Instruction>(Cond);
1599 // Compute the effective opcode for Cond, taking into account whether it needs
1600 // to be inverted, e.g.
1601 // and (not (or A, B)), C
1603 // and (and (not A, not B), C)
1606 BOpc = BOp->getOpcode();
1608 if (BOpc == Instruction::And)
1609 BOpc = Instruction::Or;
1610 else if (BOpc == Instruction::Or)
1611 BOpc = Instruction::And;
1615 // If this node is not part of the or/and tree, emit it as a branch.
1616 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1617 BOpc != Opc || !BOp->hasOneUse() ||
1618 BOp->getParent() != CurBB->getBasicBlock() ||
1619 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1620 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1621 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1622 TProb, FProb, InvertCond);
1626 // Create TmpBB after CurBB.
1627 MachineFunction::iterator BBI(CurBB);
1628 MachineFunction &MF = DAG.getMachineFunction();
1629 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1630 CurBB->getParent()->insert(++BBI, TmpBB);
1632 if (Opc == Instruction::Or) {
1633 // Codegen X | Y as:
1642 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1643 // The requirement is that
1644 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1645 // = TrueProb for original BB.
1646 // Assuming the original probabilities are A and B, one choice is to set
1647 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1648 // A/(1+B) and 2B/(1+B). This choice assumes that
1649 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1650 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1651 // TmpBB, but the math is more complicated.
1653 auto NewTrueProb = TProb / 2;
1654 auto NewFalseProb = TProb / 2 + FProb;
1655 // Emit the LHS condition.
1656 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1657 NewTrueProb, NewFalseProb, InvertCond);
1659 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1660 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1661 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1662 // Emit the RHS condition into TmpBB.
1663 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1664 Probs[0], Probs[1], InvertCond);
1666 assert(Opc == Instruction::And && "Unknown merge op!");
1667 // Codegen X & Y as:
1675 // This requires creation of TmpBB after CurBB.
1677 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1678 // The requirement is that
1679 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1680 // = FalseProb for original BB.
1681 // Assuming the original probabilities are A and B, one choice is to set
1682 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1683 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1684 // TrueProb for BB1 * FalseProb for TmpBB.
1686 auto NewTrueProb = TProb + FProb / 2;
1687 auto NewFalseProb = FProb / 2;
1688 // Emit the LHS condition.
1689 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1690 NewTrueProb, NewFalseProb, InvertCond);
1692 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1693 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1694 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1695 // Emit the RHS condition into TmpBB.
1696 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1697 Probs[0], Probs[1], InvertCond);
1701 /// If the set of cases should be emitted as a series of branches, return true.
1702 /// If we should emit this as a bunch of and/or'd together conditions, return
1705 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1706 if (Cases.size() != 2) return true;
1708 // If this is two comparisons of the same values or'd or and'd together, they
1709 // will get folded into a single comparison, so don't emit two blocks.
1710 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1711 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1712 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1713 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1717 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1718 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1719 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1720 Cases[0].CC == Cases[1].CC &&
1721 isa<Constant>(Cases[0].CmpRHS) &&
1722 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1723 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1725 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1732 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1733 MachineBasicBlock *BrMBB = FuncInfo.MBB;
1735 // Update machine-CFG edges.
1736 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1738 if (I.isUnconditional()) {
1739 // Update machine-CFG edges.
1740 BrMBB->addSuccessor(Succ0MBB);
1742 // If this is not a fall-through branch or optimizations are switched off,
1744 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1745 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1746 MVT::Other, getControlRoot(),
1747 DAG.getBasicBlock(Succ0MBB)));
1752 // If this condition is one of the special cases we handle, do special stuff
1754 const Value *CondVal = I.getCondition();
1755 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1757 // If this is a series of conditions that are or'd or and'd together, emit
1758 // this as a sequence of branches instead of setcc's with and/or operations.
1759 // As long as jumps are not expensive, this should improve performance.
1760 // For example, instead of something like:
1773 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1774 Instruction::BinaryOps Opcode = BOp->getOpcode();
1775 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1776 !I.getMetadata(LLVMContext::MD_unpredictable) &&
1777 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1778 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1780 getEdgeProbability(BrMBB, Succ0MBB),
1781 getEdgeProbability(BrMBB, Succ1MBB),
1782 /*InvertCond=*/false);
1783 // If the compares in later blocks need to use values not currently
1784 // exported from this block, export them now. This block should always
1785 // be the first entry.
1786 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
1788 // Allow some cases to be rejected.
1789 if (ShouldEmitAsBranches(SwitchCases)) {
1790 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1791 ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
1792 ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
1795 // Emit the branch for this block.
1796 visitSwitchCase(SwitchCases[0], BrMBB);
1797 SwitchCases.erase(SwitchCases.begin());
1801 // Okay, we decided not to do this, remove any inserted MBB's and clear
1803 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1804 FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1806 SwitchCases.clear();
1810 // Create a CaseBlock record representing this branch.
1811 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1812 nullptr, Succ0MBB, Succ1MBB, BrMBB);
1814 // Use visitSwitchCase to actually insert the fast branch sequence for this
1816 visitSwitchCase(CB, BrMBB);
1819 /// visitSwitchCase - Emits the necessary code to represent a single node in
1820 /// the binary search tree resulting from lowering a switch instruction.
1821 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
1822 MachineBasicBlock *SwitchBB) {
1824 SDValue CondLHS = getValue(CB.CmpLHS);
1825 SDLoc dl = getCurSDLoc();
1827 // Build the setcc now.
1829 // Fold "(X == true)" to X and "(X == false)" to !X to
1830 // handle common cases produced by branch lowering.
1831 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1832 CB.CC == ISD::SETEQ)
1834 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1835 CB.CC == ISD::SETEQ) {
1836 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1837 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1839 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1841 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
1843 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1844 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1846 SDValue CmpOp = getValue(CB.CmpMHS);
1847 EVT VT = CmpOp.getValueType();
1849 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1850 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1853 SDValue SUB = DAG.getNode(ISD::SUB, dl,
1854 VT, CmpOp, DAG.getConstant(Low, dl, VT));
1855 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1856 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1860 // Update successor info
1861 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
1862 // TrueBB and FalseBB are always different unless the incoming IR is
1863 // degenerate. This only happens when running llc on weird IR.
1864 if (CB.TrueBB != CB.FalseBB)
1865 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
1866 SwitchBB->normalizeSuccProbs();
1868 // If the lhs block is the next block, invert the condition so that we can
1869 // fall through to the lhs instead of the rhs block.
1870 if (CB.TrueBB == NextBlock(SwitchBB)) {
1871 std::swap(CB.TrueBB, CB.FalseBB);
1872 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1873 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1876 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1877 MVT::Other, getControlRoot(), Cond,
1878 DAG.getBasicBlock(CB.TrueBB));
1880 // Insert the false branch. Do this even if it's a fall through branch,
1881 // this makes it easier to do DAG optimizations which require inverting
1882 // the branch condition.
1883 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1884 DAG.getBasicBlock(CB.FalseBB));
1886 DAG.setRoot(BrCond);
1889 /// visitJumpTable - Emit JumpTable node in the current MBB
1890 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
1891 // Emit the code for the jump table
1892 assert(JT.Reg != -1U && "Should lower JT Header first!");
1893 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1894 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
1896 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1897 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1898 MVT::Other, Index.getValue(1),
1900 DAG.setRoot(BrJumpTable);
1903 /// visitJumpTableHeader - This function emits necessary code to produce index
1904 /// in the JumpTable from switch case.
1905 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
1906 JumpTableHeader &JTH,
1907 MachineBasicBlock *SwitchBB) {
1908 SDLoc dl = getCurSDLoc();
1910 // Subtract the lowest switch case value from the value being switched on and
1911 // conditional branch to default mbb if the result is greater than the
1912 // difference between smallest and largest cases.
1913 SDValue SwitchOp = getValue(JTH.SValue);
1914 EVT VT = SwitchOp.getValueType();
1915 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
1916 DAG.getConstant(JTH.First, dl, VT));
1918 // The SDNode we just created, which holds the value being switched on minus
1919 // the smallest case value, needs to be copied to a virtual register so it
1920 // can be used as an index into the jump table in a subsequent basic block.
1921 // This value may be smaller or larger than the target's pointer type, and
1922 // therefore require extension or truncating.
1923 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1924 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
1926 unsigned JumpTableReg =
1927 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
1928 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
1929 JumpTableReg, SwitchOp);
1930 JT.Reg = JumpTableReg;
1932 // Emit the range check for the jump table, and branch to the default block
1933 // for the switch statement if the value being switched on exceeds the largest
1934 // case in the switch.
1935 SDValue CMP = DAG.getSetCC(
1936 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1937 Sub.getValueType()),
1938 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
1940 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1941 MVT::Other, CopyTo, CMP,
1942 DAG.getBasicBlock(JT.Default));
1944 // Avoid emitting unnecessary branches to the next block.
1945 if (JT.MBB != NextBlock(SwitchBB))
1946 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1947 DAG.getBasicBlock(JT.MBB));
1949 DAG.setRoot(BrCond);
1952 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
1953 /// variable if there exists one.
1954 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
1956 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1957 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
1958 MachineFunction &MF = DAG.getMachineFunction();
1959 Value *Global = TLI.getSDagStackGuard(*MF.getFunction()->getParent());
1960 MachineSDNode *Node =
1961 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
1963 MachinePointerInfo MPInfo(Global);
1964 MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
1965 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1966 MachineMemOperand::MODereferenceable;
1967 *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
1968 DAG.getEVTAlignment(PtrTy));
1969 Node->setMemRefs(MemRefs, MemRefs + 1);
1971 return SDValue(Node, 0);
1974 /// Codegen a new tail for a stack protector check ParentMBB which has had its
1975 /// tail spliced into a stack protector check success bb.
1977 /// For a high level explanation of how this fits into the stack protector
1978 /// generation see the comment on the declaration of class
1979 /// StackProtectorDescriptor.
1980 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
1981 MachineBasicBlock *ParentBB) {
1983 // First create the loads to the guard/stack slot for the comparison.
1984 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1985 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
1987 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
1988 int FI = MFI.getStackProtectorIndex();
1991 SDLoc dl = getCurSDLoc();
1992 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
1993 const Module &M = *ParentBB->getParent()->getFunction()->getParent();
1994 unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
1996 // Generate code to load the content of the guard slot.
1997 SDValue StackSlot = DAG.getLoad(
1998 PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
1999 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2000 MachineMemOperand::MOVolatile);
2002 // Retrieve guard check function, nullptr if instrumentation is inlined.
2003 if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2004 // The target provides a guard check function to validate the guard value.
2005 // Generate a call to that function with the content of the guard slot as
2007 auto *Fn = cast<Function>(GuardCheck);
2008 FunctionType *FnTy = Fn->getFunctionType();
2009 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2011 TargetLowering::ArgListTy Args;
2012 TargetLowering::ArgListEntry Entry;
2013 Entry.Node = StackSlot;
2014 Entry.Ty = FnTy->getParamType(0);
2015 if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2016 Entry.IsInReg = true;
2017 Args.push_back(Entry);
2019 TargetLowering::CallLoweringInfo CLI(DAG);
2020 CLI.setDebugLoc(getCurSDLoc())
2021 .setChain(DAG.getEntryNode())
2022 .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2023 getValue(GuardCheck), std::move(Args));
2025 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2026 DAG.setRoot(Result.second);
2030 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2031 // Otherwise, emit a volatile load to retrieve the stack guard value.
2032 SDValue Chain = DAG.getEntryNode();
2033 if (TLI.useLoadStackGuardNode()) {
2034 Guard = getLoadStackGuard(DAG, dl, Chain);
2036 const Value *IRGuard = TLI.getSDagStackGuard(M);
2037 SDValue GuardPtr = getValue(IRGuard);
2040 DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2041 Align, MachineMemOperand::MOVolatile);
2044 // Perform the comparison via a subtract/getsetcc.
2045 EVT VT = Guard.getValueType();
2046 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
2048 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2050 Sub.getValueType()),
2051 Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2053 // If the sub is not 0, then we know the guard/stackslot do not equal, so
2054 // branch to failure MBB.
2055 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2056 MVT::Other, StackSlot.getOperand(0),
2057 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2058 // Otherwise branch to success MBB.
2059 SDValue Br = DAG.getNode(ISD::BR, dl,
2061 DAG.getBasicBlock(SPD.getSuccessMBB()));
2066 /// Codegen the failure basic block for a stack protector check.
2068 /// A failure stack protector machine basic block consists simply of a call to
2069 /// __stack_chk_fail().
2071 /// For a high level explanation of how this fits into the stack protector
2072 /// generation see the comment on the declaration of class
2073 /// StackProtectorDescriptor.
2075 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2076 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2078 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2079 None, false, getCurSDLoc(), false, false).second;
2083 /// visitBitTestHeader - This function emits necessary code to produce value
2084 /// suitable for "bit tests"
2085 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2086 MachineBasicBlock *SwitchBB) {
2087 SDLoc dl = getCurSDLoc();
2089 // Subtract the minimum value
2090 SDValue SwitchOp = getValue(B.SValue);
2091 EVT VT = SwitchOp.getValueType();
2092 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2093 DAG.getConstant(B.First, dl, VT));
2096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2097 SDValue RangeCmp = DAG.getSetCC(
2098 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2099 Sub.getValueType()),
2100 Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2102 // Determine the type of the test operands.
2103 bool UsePtrType = false;
2104 if (!TLI.isTypeLegal(VT))
2107 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2108 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2109 // Switch table case range are encoded into series of masks.
2110 // Just use pointer type, it's guaranteed to fit.
2116 VT = TLI.getPointerTy(DAG.getDataLayout());
2117 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2120 B.RegVT = VT.getSimpleVT();
2121 B.Reg = FuncInfo.CreateReg(B.RegVT);
2122 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2124 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2126 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2127 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2128 SwitchBB->normalizeSuccProbs();
2130 SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2131 MVT::Other, CopyTo, RangeCmp,
2132 DAG.getBasicBlock(B.Default));
2134 // Avoid emitting unnecessary branches to the next block.
2135 if (MBB != NextBlock(SwitchBB))
2136 BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2137 DAG.getBasicBlock(MBB));
2139 DAG.setRoot(BrRange);
2142 /// visitBitTestCase - this function produces one "bit test"
2143 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2144 MachineBasicBlock* NextMBB,
2145 BranchProbability BranchProbToNext,
2148 MachineBasicBlock *SwitchBB) {
2149 SDLoc dl = getCurSDLoc();
2151 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2153 unsigned PopCount = countPopulation(B.Mask);
2154 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2155 if (PopCount == 1) {
2156 // Testing for a single bit; just compare the shift count with what it
2157 // would need to be to shift a 1 bit in that position.
2159 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2160 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2162 } else if (PopCount == BB.Range) {
2163 // There is only one zero bit in the range, test for it directly.
2165 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2166 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2169 // Make desired shift
2170 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2171 DAG.getConstant(1, dl, VT), ShiftOp);
2173 // Emit bit tests and jumps
2174 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2175 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2177 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2178 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2181 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2182 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2183 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2184 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2185 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2186 // one as they are relative probabilities (and thus work more like weights),
2187 // and hence we need to normalize them to let the sum of them become one.
2188 SwitchBB->normalizeSuccProbs();
2190 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2191 MVT::Other, getControlRoot(),
2192 Cmp, DAG.getBasicBlock(B.TargetBB));
2194 // Avoid emitting unnecessary branches to the next block.
2195 if (NextMBB != NextBlock(SwitchBB))
2196 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2197 DAG.getBasicBlock(NextMBB));
2202 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2203 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2205 // Retrieve successors. Look through artificial IR level blocks like
2206 // catchswitch for successors.
2207 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2208 const BasicBlock *EHPadBB = I.getSuccessor(1);
2210 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2211 // have to do anything here to lower funclet bundles.
2212 assert(!I.hasOperandBundlesOtherThan(
2213 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2214 "Cannot lower invokes with arbitrary operand bundles yet!");
2216 const Value *Callee(I.getCalledValue());
2217 const Function *Fn = dyn_cast<Function>(Callee);
2218 if (isa<InlineAsm>(Callee))
2220 else if (Fn && Fn->isIntrinsic()) {
2221 switch (Fn->getIntrinsicID()) {
2223 llvm_unreachable("Cannot invoke this intrinsic");
2224 case Intrinsic::donothing:
2225 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2227 case Intrinsic::experimental_patchpoint_void:
2228 case Intrinsic::experimental_patchpoint_i64:
2229 visitPatchpoint(&I, EHPadBB);
2231 case Intrinsic::experimental_gc_statepoint:
2232 LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2235 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2236 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2237 // Eventually we will support lowering the @llvm.experimental.deoptimize
2238 // intrinsic, and right now there are no plans to support other intrinsics
2239 // with deopt state.
2240 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2242 LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2245 // If the value of the invoke is used outside of its defining block, make it
2246 // available as a virtual register.
2247 // We already took care of the exported value for the statepoint instruction
2248 // during call to the LowerStatepoint.
2249 if (!isStatepoint(I)) {
2250 CopyToExportRegsIfNeeded(&I);
2253 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2254 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2255 BranchProbability EHPadBBProb =
2256 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2257 : BranchProbability::getZero();
2258 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2260 // Update successor info.
2261 addSuccessorWithProb(InvokeMBB, Return);
2262 for (auto &UnwindDest : UnwindDests) {
2263 UnwindDest.first->setIsEHPad();
2264 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2266 InvokeMBB->normalizeSuccProbs();
2268 // Drop into normal successor.
2269 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2270 MVT::Other, getControlRoot(),
2271 DAG.getBasicBlock(Return)));
2274 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2275 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2278 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2279 assert(FuncInfo.MBB->isEHPad() &&
2280 "Call to landingpad not in landing pad!");
2282 MachineBasicBlock *MBB = FuncInfo.MBB;
2283 addLandingPadInfo(LP, *MBB);
2285 // If there aren't registers to copy the values into (e.g., during SjLj
2286 // exceptions), then don't bother to create these DAG nodes.
2287 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2288 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2289 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2290 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2293 // If landingpad's return type is token type, we don't create DAG nodes
2294 // for its exception pointer and selector value. The extraction of exception
2295 // pointer or selector value from token type landingpads is not currently
2297 if (LP.getType()->isTokenTy())
2300 SmallVector<EVT, 2> ValueVTs;
2301 SDLoc dl = getCurSDLoc();
2302 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2303 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2305 // Get the two live-in registers as SDValues. The physregs have already been
2306 // copied into virtual registers.
2308 if (FuncInfo.ExceptionPointerVirtReg) {
2309 Ops[0] = DAG.getZExtOrTrunc(
2310 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2311 FuncInfo.ExceptionPointerVirtReg,
2312 TLI.getPointerTy(DAG.getDataLayout())),
2315 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2317 Ops[1] = DAG.getZExtOrTrunc(
2318 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2319 FuncInfo.ExceptionSelectorVirtReg,
2320 TLI.getPointerTy(DAG.getDataLayout())),
2324 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2325 DAG.getVTList(ValueVTs), Ops);
2329 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2331 for (const CaseCluster &CC : Clusters)
2332 assert(CC.Low == CC.High && "Input clusters must be single-case");
2335 std::sort(Clusters.begin(), Clusters.end(),
2336 [](const CaseCluster &a, const CaseCluster &b) {
2337 return a.Low->getValue().slt(b.Low->getValue());
2340 // Merge adjacent clusters with the same destination.
2341 const unsigned N = Clusters.size();
2342 unsigned DstIndex = 0;
2343 for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2344 CaseCluster &CC = Clusters[SrcIndex];
2345 const ConstantInt *CaseVal = CC.Low;
2346 MachineBasicBlock *Succ = CC.MBB;
2348 if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2349 (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2350 // If this case has the same successor and is a neighbour, merge it into
2351 // the previous cluster.
2352 Clusters[DstIndex - 1].High = CaseVal;
2353 Clusters[DstIndex - 1].Prob += CC.Prob;
2355 std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2356 sizeof(Clusters[SrcIndex]));
2359 Clusters.resize(DstIndex);
2362 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2363 MachineBasicBlock *Last) {
2365 for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2366 if (JTCases[i].first.HeaderBB == First)
2367 JTCases[i].first.HeaderBB = Last;
2369 // Update BitTestCases.
2370 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2371 if (BitTestCases[i].Parent == First)
2372 BitTestCases[i].Parent = Last;
2375 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2376 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2378 // Update machine-CFG edges with unique successors.
2379 SmallSet<BasicBlock*, 32> Done;
2380 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2381 BasicBlock *BB = I.getSuccessor(i);
2382 bool Inserted = Done.insert(BB).second;
2386 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2387 addSuccessorWithProb(IndirectBrMBB, Succ);
2389 IndirectBrMBB->normalizeSuccProbs();
2391 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2392 MVT::Other, getControlRoot(),
2393 getValue(I.getAddress())));
2396 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2397 if (DAG.getTarget().Options.TrapUnreachable)
2399 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2402 void SelectionDAGBuilder::visitFSub(const User &I) {
2403 // -0.0 - X --> fneg
2404 Type *Ty = I.getType();
2405 if (isa<Constant>(I.getOperand(0)) &&
2406 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2407 SDValue Op2 = getValue(I.getOperand(1));
2408 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2409 Op2.getValueType(), Op2));
2413 visitBinary(I, ISD::FSUB);
2416 /// Checks if the given instruction performs a vector reduction, in which case
2417 /// we have the freedom to alter the elements in the result as long as the
2418 /// reduction of them stays unchanged.
2419 static bool isVectorReductionOp(const User *I) {
2420 const Instruction *Inst = dyn_cast<Instruction>(I);
2421 if (!Inst || !Inst->getType()->isVectorTy())
2424 auto OpCode = Inst->getOpcode();
2426 case Instruction::Add:
2427 case Instruction::Mul:
2428 case Instruction::And:
2429 case Instruction::Or:
2430 case Instruction::Xor:
2432 case Instruction::FAdd:
2433 case Instruction::FMul:
2434 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2435 if (FPOp->getFastMathFlags().unsafeAlgebra())
2442 unsigned ElemNum = Inst->getType()->getVectorNumElements();
2443 unsigned ElemNumToReduce = ElemNum;
2445 // Do DFS search on the def-use chain from the given instruction. We only
2446 // allow four kinds of operations during the search until we reach the
2447 // instruction that extracts the first element from the vector:
2449 // 1. The reduction operation of the same opcode as the given instruction.
2453 // 3. ShuffleVector instruction together with a reduction operation that
2454 // does a partial reduction.
2456 // 4. ExtractElement that extracts the first element from the vector, and we
2457 // stop searching the def-use chain here.
2459 // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2460 // from 1-3 to the stack to continue the DFS. The given instruction is not
2461 // a reduction operation if we meet any other instructions other than those
2464 SmallVector<const User *, 16> UsersToVisit{Inst};
2465 SmallPtrSet<const User *, 16> Visited;
2466 bool ReduxExtracted = false;
2468 while (!UsersToVisit.empty()) {
2469 auto User = UsersToVisit.back();
2470 UsersToVisit.pop_back();
2471 if (!Visited.insert(User).second)
2474 for (const auto &U : User->users()) {
2475 auto Inst = dyn_cast<Instruction>(U);
2479 if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2480 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2481 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().unsafeAlgebra())
2483 UsersToVisit.push_back(U);
2484 } else if (const ShuffleVectorInst *ShufInst =
2485 dyn_cast<ShuffleVectorInst>(U)) {
2486 // Detect the following pattern: A ShuffleVector instruction together
2487 // with a reduction that do partial reduction on the first and second
2488 // ElemNumToReduce / 2 elements, and store the result in
2489 // ElemNumToReduce / 2 elements in another vector.
2491 unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2492 if (ResultElements < ElemNum)
2495 if (ElemNumToReduce == 1)
2497 if (!isa<UndefValue>(U->getOperand(1)))
2499 for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2500 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2502 for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2503 if (ShufInst->getMaskValue(i) != -1)
2506 // There is only one user of this ShuffleVector instruction, which
2507 // must be a reduction operation.
2508 if (!U->hasOneUse())
2511 auto U2 = dyn_cast<Instruction>(*U->user_begin());
2512 if (!U2 || U2->getOpcode() != OpCode)
2515 // Check operands of the reduction operation.
2516 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2517 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2518 UsersToVisit.push_back(U2);
2519 ElemNumToReduce /= 2;
2522 } else if (isa<ExtractElementInst>(U)) {
2523 // At this moment we should have reduced all elements in the vector.
2524 if (ElemNumToReduce != 1)
2527 const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2528 if (!Val || Val->getZExtValue() != 0)
2531 ReduxExtracted = true;
2536 return ReduxExtracted;
2539 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2540 SDValue Op1 = getValue(I.getOperand(0));
2541 SDValue Op2 = getValue(I.getOperand(1));
2546 bool vec_redux = false;
2549 if (const OverflowingBinaryOperator *OFBinOp =
2550 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2551 nuw = OFBinOp->hasNoUnsignedWrap();
2552 nsw = OFBinOp->hasNoSignedWrap();
2554 if (const PossiblyExactOperator *ExactOp =
2555 dyn_cast<const PossiblyExactOperator>(&I))
2556 exact = ExactOp->isExact();
2557 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2558 FMF = FPOp->getFastMathFlags();
2560 if (isVectorReductionOp(&I)) {
2562 DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2566 Flags.setExact(exact);
2567 Flags.setNoSignedWrap(nsw);
2568 Flags.setNoUnsignedWrap(nuw);
2569 Flags.setVectorReduction(vec_redux);
2570 Flags.setAllowReciprocal(FMF.allowReciprocal());
2571 Flags.setAllowContract(FMF.allowContract());
2572 Flags.setNoInfs(FMF.noInfs());
2573 Flags.setNoNaNs(FMF.noNaNs());
2574 Flags.setNoSignedZeros(FMF.noSignedZeros());
2575 Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2577 SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2579 setValue(&I, BinNodeValue);
2582 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2583 SDValue Op1 = getValue(I.getOperand(0));
2584 SDValue Op2 = getValue(I.getOperand(1));
2586 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2587 Op2.getValueType(), DAG.getDataLayout());
2589 // Coerce the shift amount to the right type if we can.
2590 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2591 unsigned ShiftSize = ShiftTy.getSizeInBits();
2592 unsigned Op2Size = Op2.getValueSizeInBits();
2593 SDLoc DL = getCurSDLoc();
2595 // If the operand is smaller than the shift count type, promote it.
2596 if (ShiftSize > Op2Size)
2597 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2599 // If the operand is larger than the shift count type but the shift
2600 // count type has enough bits to represent any shift value, truncate
2601 // it now. This is a common case and it exposes the truncate to
2602 // optimization early.
2603 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2604 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2605 // Otherwise we'll need to temporarily settle for some other convenient
2606 // type. Type legalization will make adjustments once the shiftee is split.
2608 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2615 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2617 if (const OverflowingBinaryOperator *OFBinOp =
2618 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2619 nuw = OFBinOp->hasNoUnsignedWrap();
2620 nsw = OFBinOp->hasNoSignedWrap();
2622 if (const PossiblyExactOperator *ExactOp =
2623 dyn_cast<const PossiblyExactOperator>(&I))
2624 exact = ExactOp->isExact();
2627 Flags.setExact(exact);
2628 Flags.setNoSignedWrap(nsw);
2629 Flags.setNoUnsignedWrap(nuw);
2630 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2635 void SelectionDAGBuilder::visitSDiv(const User &I) {
2636 SDValue Op1 = getValue(I.getOperand(0));
2637 SDValue Op2 = getValue(I.getOperand(1));
2640 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2641 cast<PossiblyExactOperator>(&I)->isExact());
2642 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2646 void SelectionDAGBuilder::visitICmp(const User &I) {
2647 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2648 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2649 predicate = IC->getPredicate();
2650 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2651 predicate = ICmpInst::Predicate(IC->getPredicate());
2652 SDValue Op1 = getValue(I.getOperand(0));
2653 SDValue Op2 = getValue(I.getOperand(1));
2654 ISD::CondCode Opcode = getICmpCondCode(predicate);
2656 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2658 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2661 void SelectionDAGBuilder::visitFCmp(const User &I) {
2662 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2663 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2664 predicate = FC->getPredicate();
2665 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2666 predicate = FCmpInst::Predicate(FC->getPredicate());
2667 SDValue Op1 = getValue(I.getOperand(0));
2668 SDValue Op2 = getValue(I.getOperand(1));
2669 ISD::CondCode Condition = getFCmpCondCode(predicate);
2671 // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
2672 // FIXME: We should propagate the fast-math-flags to the DAG node itself for
2673 // further optimization, but currently FMF is only applicable to binary nodes.
2674 if (TM.Options.NoNaNsFPMath)
2675 Condition = getFCmpCodeWithoutNaN(Condition);
2676 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2678 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2681 // Check if the condition of the select has one use or two users that are both
2682 // selects with the same condition.
2683 static bool hasOnlySelectUsers(const Value *Cond) {
2684 return all_of(Cond->users(), [](const Value *V) {
2685 return isa<SelectInst>(V);
2689 void SelectionDAGBuilder::visitSelect(const User &I) {
2690 SmallVector<EVT, 4> ValueVTs;
2691 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2693 unsigned NumValues = ValueVTs.size();
2694 if (NumValues == 0) return;
2696 SmallVector<SDValue, 4> Values(NumValues);
2697 SDValue Cond = getValue(I.getOperand(0));
2698 SDValue LHSVal = getValue(I.getOperand(1));
2699 SDValue RHSVal = getValue(I.getOperand(2));
2700 auto BaseOps = {Cond};
2701 ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2702 ISD::VSELECT : ISD::SELECT;
2704 // Min/max matching is only viable if all output VTs are the same.
2705 if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2706 EVT VT = ValueVTs[0];
2707 LLVMContext &Ctx = *DAG.getContext();
2708 auto &TLI = DAG.getTargetLoweringInfo();
2710 // We care about the legality of the operation after it has been type
2712 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2713 VT != TLI.getTypeToTransformTo(Ctx, VT))
2714 VT = TLI.getTypeToTransformTo(Ctx, VT);
2716 // If the vselect is legal, assume we want to leave this as a vector setcc +
2717 // vselect. Otherwise, if this is going to be scalarized, we want to see if
2718 // min/max is legal on the scalar type.
2719 bool UseScalarMinMax = VT.isVector() &&
2720 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2723 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2724 ISD::NodeType Opc = ISD::DELETED_NODE;
2725 switch (SPR.Flavor) {
2726 case SPF_UMAX: Opc = ISD::UMAX; break;
2727 case SPF_UMIN: Opc = ISD::UMIN; break;
2728 case SPF_SMAX: Opc = ISD::SMAX; break;
2729 case SPF_SMIN: Opc = ISD::SMIN; break;
2731 switch (SPR.NaNBehavior) {
2732 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2733 case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2734 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2735 case SPNB_RETURNS_ANY: {
2736 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2738 else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2740 else if (UseScalarMinMax)
2741 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2742 ISD::FMINNUM : ISD::FMINNAN;
2748 switch (SPR.NaNBehavior) {
2749 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2750 case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2751 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2752 case SPNB_RETURNS_ANY:
2754 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2756 else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2758 else if (UseScalarMinMax)
2759 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2760 ISD::FMAXNUM : ISD::FMAXNAN;
2767 if (Opc != ISD::DELETED_NODE &&
2768 (TLI.isOperationLegalOrCustom(Opc, VT) ||
2770 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2771 // If the underlying comparison instruction is used by any other
2772 // instruction, the consumed instructions won't be destroyed, so it is
2773 // not profitable to convert to a min/max.
2774 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2776 LHSVal = getValue(LHS);
2777 RHSVal = getValue(RHS);
2782 for (unsigned i = 0; i != NumValues; ++i) {
2783 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2784 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2785 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2786 Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2787 LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2791 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
2792 DAG.getVTList(ValueVTs), Values));
2795 void SelectionDAGBuilder::visitTrunc(const User &I) {
2796 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2797 SDValue N = getValue(I.getOperand(0));
2798 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2800 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2803 void SelectionDAGBuilder::visitZExt(const User &I) {
2804 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2805 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2806 SDValue N = getValue(I.getOperand(0));
2807 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2809 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2812 void SelectionDAGBuilder::visitSExt(const User &I) {
2813 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2814 // SExt also can't be a cast to bool for same reason. So, nothing much to do
2815 SDValue N = getValue(I.getOperand(0));
2816 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2818 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2821 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2822 // FPTrunc is never a no-op cast, no need to check
2823 SDValue N = getValue(I.getOperand(0));
2824 SDLoc dl = getCurSDLoc();
2825 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2826 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2827 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2828 DAG.getTargetConstant(
2829 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2832 void SelectionDAGBuilder::visitFPExt(const User &I) {
2833 // FPExt is never a no-op cast, no need to check
2834 SDValue N = getValue(I.getOperand(0));
2835 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2837 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2840 void SelectionDAGBuilder::visitFPToUI(const User &I) {
2841 // FPToUI is never a no-op cast, no need to check
2842 SDValue N = getValue(I.getOperand(0));
2843 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2845 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2848 void SelectionDAGBuilder::visitFPToSI(const User &I) {
2849 // FPToSI is never a no-op cast, no need to check
2850 SDValue N = getValue(I.getOperand(0));
2851 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2853 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2856 void SelectionDAGBuilder::visitUIToFP(const User &I) {
2857 // UIToFP is never a no-op cast, no need to check
2858 SDValue N = getValue(I.getOperand(0));
2859 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2861 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2864 void SelectionDAGBuilder::visitSIToFP(const User &I) {
2865 // SIToFP is never a no-op cast, no need to check
2866 SDValue N = getValue(I.getOperand(0));
2867 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2869 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2872 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2873 // What to do depends on the size of the integer and the size of the pointer.
2874 // We can either truncate, zero extend, or no-op, accordingly.
2875 SDValue N = getValue(I.getOperand(0));
2876 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2878 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2881 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2882 // What to do depends on the size of the integer and the size of the pointer.
2883 // We can either truncate, zero extend, or no-op, accordingly.
2884 SDValue N = getValue(I.getOperand(0));
2885 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2887 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2890 void SelectionDAGBuilder::visitBitCast(const User &I) {
2891 SDValue N = getValue(I.getOperand(0));
2892 SDLoc dl = getCurSDLoc();
2893 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2896 // BitCast assures us that source and destination are the same size so this is
2897 // either a BITCAST or a no-op.
2898 if (DestVT != N.getValueType())
2899 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2900 DestVT, N)); // convert types.
2901 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
2902 // might fold any kind of constant expression to an integer constant and that
2903 // is not what we are looking for. Only recognize a bitcast of a genuine
2904 // constant integer as an opaque constant.
2905 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
2906 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
2909 setValue(&I, N); // noop cast.
2912 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
2913 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2914 const Value *SV = I.getOperand(0);
2915 SDValue N = getValue(SV);
2916 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2918 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
2919 unsigned DestAS = I.getType()->getPointerAddressSpace();
2921 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
2922 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
2927 void SelectionDAGBuilder::visitInsertElement(const User &I) {
2928 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2929 SDValue InVec = getValue(I.getOperand(0));
2930 SDValue InVal = getValue(I.getOperand(1));
2931 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
2932 TLI.getVectorIdxTy(DAG.getDataLayout()));
2933 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
2934 TLI.getValueType(DAG.getDataLayout(), I.getType()),
2935 InVec, InVal, InIdx));
2938 void SelectionDAGBuilder::visitExtractElement(const User &I) {
2939 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2940 SDValue InVec = getValue(I.getOperand(0));
2941 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
2942 TLI.getVectorIdxTy(DAG.getDataLayout()));
2943 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
2944 TLI.getValueType(DAG.getDataLayout(), I.getType()),
2948 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
2949 SDValue Src1 = getValue(I.getOperand(0));
2950 SDValue Src2 = getValue(I.getOperand(1));
2951 SDLoc DL = getCurSDLoc();
2953 SmallVector<int, 8> Mask;
2954 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
2955 unsigned MaskNumElts = Mask.size();
2957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2958 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2959 EVT SrcVT = Src1.getValueType();
2960 unsigned SrcNumElts = SrcVT.getVectorNumElements();
2962 if (SrcNumElts == MaskNumElts) {
2963 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
2967 // Normalize the shuffle vector since mask and vector length don't match.
2968 if (SrcNumElts < MaskNumElts) {
2969 // Mask is longer than the source vectors. We can use concatenate vector to
2970 // make the mask and vectors lengths match.
2972 if (MaskNumElts % SrcNumElts == 0) {
2973 // Mask length is a multiple of the source vector length.
2974 // Check if the shuffle is some kind of concatenation of the input
2976 unsigned NumConcat = MaskNumElts / SrcNumElts;
2977 bool IsConcat = true;
2978 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
2979 for (unsigned i = 0; i != MaskNumElts; ++i) {
2983 // Ensure the indices in each SrcVT sized piece are sequential and that
2984 // the same source is used for the whole piece.
2985 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
2986 (ConcatSrcs[i / SrcNumElts] >= 0 &&
2987 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
2991 // Remember which source this index came from.
2992 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
2995 // The shuffle is concatenating multiple vectors together. Just emit
2996 // a CONCAT_VECTORS operation.
2998 SmallVector<SDValue, 8> ConcatOps;
2999 for (auto Src : ConcatSrcs) {
3001 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3003 ConcatOps.push_back(Src1);
3005 ConcatOps.push_back(Src2);
3007 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3012 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3013 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3014 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3017 // Pad both vectors with undefs to make them the same length as the mask.
3018 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3020 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3021 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3025 Src1 = Src1.isUndef()
3026 ? DAG.getUNDEF(PaddedVT)
3027 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3028 Src2 = Src2.isUndef()
3029 ? DAG.getUNDEF(PaddedVT)
3030 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3032 // Readjust mask for new input vector length.
3033 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3034 for (unsigned i = 0; i != MaskNumElts; ++i) {
3036 if (Idx >= (int)SrcNumElts)
3037 Idx -= SrcNumElts - PaddedMaskNumElts;
3041 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3043 // If the concatenated vector was padded, extract a subvector with the
3044 // correct number of elements.
3045 if (MaskNumElts != PaddedMaskNumElts)
3046 Result = DAG.getNode(
3047 ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3048 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3050 setValue(&I, Result);
3054 if (SrcNumElts > MaskNumElts) {
3055 // Analyze the access pattern of the vector to see if we can extract
3056 // two subvectors and do the shuffle.
3057 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3058 bool CanExtract = true;
3059 for (int Idx : Mask) {
3064 if (Idx >= (int)SrcNumElts) {
3069 // If all the indices come from the same MaskNumElts sized portion of
3070 // the sources we can use extract. Also make sure the extract wouldn't
3071 // extract past the end of the source.
3072 int NewStartIdx = alignDown(Idx, MaskNumElts);
3073 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3074 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3076 // Make sure we always update StartIdx as we use it to track if all
3077 // elements are undef.
3078 StartIdx[Input] = NewStartIdx;
3081 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3082 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3086 // Extract appropriate subvector and generate a vector shuffle
3087 for (unsigned Input = 0; Input < 2; ++Input) {
3088 SDValue &Src = Input == 0 ? Src1 : Src2;
3089 if (StartIdx[Input] < 0)
3090 Src = DAG.getUNDEF(VT);
3093 ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3094 DAG.getConstant(StartIdx[Input], DL,
3095 TLI.getVectorIdxTy(DAG.getDataLayout())));
3099 // Calculate new mask.
3100 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3101 for (int &Idx : MappedOps) {
3102 if (Idx >= (int)SrcNumElts)
3103 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3108 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3113 // We can't use either concat vectors or extract subvectors so fall back to
3114 // replacing the shuffle with extract and build vector.
3115 // to insert and build vector.
3116 EVT EltVT = VT.getVectorElementType();
3117 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3118 SmallVector<SDValue,8> Ops;
3119 for (int Idx : Mask) {
3123 Res = DAG.getUNDEF(EltVT);
3125 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3126 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3128 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3129 EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3135 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3138 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3139 const Value *Op0 = I.getOperand(0);
3140 const Value *Op1 = I.getOperand(1);
3141 Type *AggTy = I.getType();
3142 Type *ValTy = Op1->getType();
3143 bool IntoUndef = isa<UndefValue>(Op0);
3144 bool FromUndef = isa<UndefValue>(Op1);
3146 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3148 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3149 SmallVector<EVT, 4> AggValueVTs;
3150 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3151 SmallVector<EVT, 4> ValValueVTs;
3152 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3154 unsigned NumAggValues = AggValueVTs.size();
3155 unsigned NumValValues = ValValueVTs.size();
3156 SmallVector<SDValue, 4> Values(NumAggValues);
3158 // Ignore an insertvalue that produces an empty object
3159 if (!NumAggValues) {
3160 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3164 SDValue Agg = getValue(Op0);
3166 // Copy the beginning value(s) from the original aggregate.
3167 for (; i != LinearIndex; ++i)
3168 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3169 SDValue(Agg.getNode(), Agg.getResNo() + i);
3170 // Copy values from the inserted value(s).
3172 SDValue Val = getValue(Op1);
3173 for (; i != LinearIndex + NumValValues; ++i)
3174 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3175 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3177 // Copy remaining value(s) from the original aggregate.
3178 for (; i != NumAggValues; ++i)
3179 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3180 SDValue(Agg.getNode(), Agg.getResNo() + i);
3182 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3183 DAG.getVTList(AggValueVTs), Values));
3186 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3187 const Value *Op0 = I.getOperand(0);
3188 Type *AggTy = Op0->getType();
3189 Type *ValTy = I.getType();
3190 bool OutOfUndef = isa<UndefValue>(Op0);
3192 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3194 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3195 SmallVector<EVT, 4> ValValueVTs;
3196 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3198 unsigned NumValValues = ValValueVTs.size();
3200 // Ignore a extractvalue that produces an empty object
3201 if (!NumValValues) {
3202 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3206 SmallVector<SDValue, 4> Values(NumValValues);
3208 SDValue Agg = getValue(Op0);
3209 // Copy out the selected value(s).
3210 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3211 Values[i - LinearIndex] =
3213 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3214 SDValue(Agg.getNode(), Agg.getResNo() + i);
3216 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3217 DAG.getVTList(ValValueVTs), Values));
3220 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3221 Value *Op0 = I.getOperand(0);
3222 // Note that the pointer operand may be a vector of pointers. Take the scalar
3223 // element which holds a pointer.
3224 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3225 SDValue N = getValue(Op0);
3226 SDLoc dl = getCurSDLoc();
3228 // Normalize Vector GEP - all scalar operands should be converted to the
3230 unsigned VectorWidth = I.getType()->isVectorTy() ?
3231 cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3233 if (VectorWidth && !N.getValueType().isVector()) {
3234 LLVMContext &Context = *DAG.getContext();
3235 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3236 N = DAG.getSplatBuildVector(VT, dl, N);
3239 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3241 const Value *Idx = GTI.getOperand();
3242 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3243 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3246 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3248 // In an inbounds GEP with an offset that is nonnegative even when
3249 // interpreted as signed, assume there is no unsigned overflow.
3251 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3252 Flags.setNoUnsignedWrap(true);
3254 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3255 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3259 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
3260 unsigned PtrSize = PtrTy.getSizeInBits();
3261 APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3263 // If this is a scalar constant or a splat vector of constants,
3264 // handle it quickly.
3265 const auto *CI = dyn_cast<ConstantInt>(Idx);
3266 if (!CI && isa<ConstantDataVector>(Idx) &&
3267 cast<ConstantDataVector>(Idx)->getSplatValue())
3268 CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3273 APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
3274 LLVMContext &Context = *DAG.getContext();
3275 SDValue OffsVal = VectorWidth ?
3276 DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, PtrTy, VectorWidth)) :
3277 DAG.getConstant(Offs, dl, PtrTy);
3279 // In an inbouds GEP with an offset that is nonnegative even when
3280 // interpreted as signed, assume there is no unsigned overflow.
3282 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3283 Flags.setNoUnsignedWrap(true);
3285 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3289 // N = N + Idx * ElementSize;
3290 SDValue IdxN = getValue(Idx);
3292 if (!IdxN.getValueType().isVector() && VectorWidth) {
3293 MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
3294 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3297 // If the index is smaller or larger than intptr_t, truncate or extend
3299 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3301 // If this is a multiply by a power of two, turn it into a shl
3302 // immediately. This is a very common case.
3303 if (ElementSize != 1) {
3304 if (ElementSize.isPowerOf2()) {
3305 unsigned Amt = ElementSize.logBase2();
3306 IdxN = DAG.getNode(ISD::SHL, dl,
3307 N.getValueType(), IdxN,
3308 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3310 SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3311 IdxN = DAG.getNode(ISD::MUL, dl,
3312 N.getValueType(), IdxN, Scale);
3316 N = DAG.getNode(ISD::ADD, dl,
3317 N.getValueType(), N, IdxN);
3324 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3325 // If this is a fixed sized alloca in the entry block of the function,
3326 // allocate it statically on the stack.
3327 if (FuncInfo.StaticAllocaMap.count(&I))
3328 return; // getValue will auto-populate this.
3330 SDLoc dl = getCurSDLoc();
3331 Type *Ty = I.getAllocatedType();
3332 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3333 auto &DL = DAG.getDataLayout();
3334 uint64_t TySize = DL.getTypeAllocSize(Ty);
3336 std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3338 SDValue AllocSize = getValue(I.getArraySize());
3340 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
3341 if (AllocSize.getValueType() != IntPtr)
3342 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3344 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3346 DAG.getConstant(TySize, dl, IntPtr));
3348 // Handle alignment. If the requested alignment is less than or equal to
3349 // the stack alignment, ignore it. If the size is greater than or equal to
3350 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3351 unsigned StackAlign =
3352 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3353 if (Align <= StackAlign)
3356 // Round the size of the allocation up to the stack alignment size
3357 // by add SA-1 to the size. This doesn't overflow because we're computing
3358 // an address inside an alloca.
3360 Flags.setNoUnsignedWrap(true);
3361 AllocSize = DAG.getNode(ISD::ADD, dl,
3362 AllocSize.getValueType(), AllocSize,
3363 DAG.getIntPtrConstant(StackAlign - 1, dl), Flags);
3365 // Mask out the low bits for alignment purposes.
3366 AllocSize = DAG.getNode(ISD::AND, dl,
3367 AllocSize.getValueType(), AllocSize,
3368 DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
3371 SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
3372 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3373 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3375 DAG.setRoot(DSA.getValue(1));
3377 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3380 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3382 return visitAtomicLoad(I);
3384 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3385 const Value *SV = I.getOperand(0);
3386 if (TLI.supportSwiftError()) {
3387 // Swifterror values can come from either a function parameter with
3388 // swifterror attribute or an alloca with swifterror attribute.
3389 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3390 if (Arg->hasSwiftErrorAttr())
3391 return visitLoadFromSwiftError(I);
3394 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3395 if (Alloca->isSwiftError())
3396 return visitLoadFromSwiftError(I);
3400 SDValue Ptr = getValue(SV);
3402 Type *Ty = I.getType();
3404 bool isVolatile = I.isVolatile();
3405 bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3406 bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3407 bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3408 unsigned Alignment = I.getAlignment();
3411 I.getAAMetadata(AAInfo);
3412 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3414 SmallVector<EVT, 4> ValueVTs;
3415 SmallVector<uint64_t, 4> Offsets;
3416 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3417 unsigned NumValues = ValueVTs.size();
3422 bool ConstantMemory = false;
3423 if (isVolatile || NumValues > MaxParallelChains)
3424 // Serialize volatile loads with other side effects.
3426 else if (AA->pointsToConstantMemory(MemoryLocation(
3427 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3428 // Do not serialize (non-volatile) loads of constant memory with anything.
3429 Root = DAG.getEntryNode();
3430 ConstantMemory = true;
3432 // Do not serialize non-volatile loads against each other.
3433 Root = DAG.getRoot();
3436 SDLoc dl = getCurSDLoc();
3439 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3441 // An aggregate load cannot wrap around the address space, so offsets to its
3442 // parts don't wrap either.
3444 Flags.setNoUnsignedWrap(true);
3446 SmallVector<SDValue, 4> Values(NumValues);
3447 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3448 EVT PtrVT = Ptr.getValueType();
3449 unsigned ChainI = 0;
3450 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3451 // Serializing loads here may result in excessive register pressure, and
3452 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3453 // could recover a bit by hoisting nodes upward in the chain by recognizing
3454 // they are side-effect free or do not alias. The optimizer should really
3455 // avoid this case by converting large object/array copies to llvm.memcpy
3456 // (MaxParallelChains should always remain as failsafe).
3457 if (ChainI == MaxParallelChains) {
3458 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3459 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3460 makeArrayRef(Chains.data(), ChainI));
3464 SDValue A = DAG.getNode(ISD::ADD, dl,
3466 DAG.getConstant(Offsets[i], dl, PtrVT),
3468 auto MMOFlags = MachineMemOperand::MONone;
3470 MMOFlags |= MachineMemOperand::MOVolatile;
3472 MMOFlags |= MachineMemOperand::MONonTemporal;
3474 MMOFlags |= MachineMemOperand::MOInvariant;
3475 if (isDereferenceable)
3476 MMOFlags |= MachineMemOperand::MODereferenceable;
3478 SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3479 MachinePointerInfo(SV, Offsets[i]), Alignment,
3480 MMOFlags, AAInfo, Ranges);
3483 Chains[ChainI] = L.getValue(1);
3486 if (!ConstantMemory) {
3487 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3488 makeArrayRef(Chains.data(), ChainI));
3492 PendingLoads.push_back(Chain);
3495 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3496 DAG.getVTList(ValueVTs), Values));
3499 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3500 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3501 assert(TLI.supportSwiftError() &&
3502 "call visitStoreToSwiftError when backend supports swifterror");
3504 SmallVector<EVT, 4> ValueVTs;
3505 SmallVector<uint64_t, 4> Offsets;
3506 const Value *SrcV = I.getOperand(0);
3507 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3508 SrcV->getType(), ValueVTs, &Offsets);
3509 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3510 "expect a single EVT for swifterror");
3512 SDValue Src = getValue(SrcV);
3513 // Create a virtual register, then update the virtual register.
3514 auto &DL = DAG.getDataLayout();
3515 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3516 unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
3517 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3518 // Chain can be getRoot or getControlRoot.
3519 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3520 SDValue(Src.getNode(), Src.getResNo()));
3521 DAG.setRoot(CopyNode);
3522 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3525 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3526 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3527 "call visitLoadFromSwiftError when backend supports swifterror");
3529 assert(!I.isVolatile() &&
3530 I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3531 I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&
3532 "Support volatile, non temporal, invariant for load_from_swift_error");
3534 const Value *SV = I.getOperand(0);
3535 Type *Ty = I.getType();
3537 I.getAAMetadata(AAInfo);
3538 assert(!AA->pointsToConstantMemory(MemoryLocation(
3539 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo)) &&
3540 "load_from_swift_error should not be constant memory");
3542 SmallVector<EVT, 4> ValueVTs;
3543 SmallVector<uint64_t, 4> Offsets;
3544 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3545 ValueVTs, &Offsets);
3546 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3547 "expect a single EVT for swifterror");
3549 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3550 SDValue L = DAG.getCopyFromReg(
3551 getRoot(), getCurSDLoc(),
3552 FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, SV), ValueVTs[0]);
3557 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3559 return visitAtomicStore(I);
3561 const Value *SrcV = I.getOperand(0);
3562 const Value *PtrV = I.getOperand(1);
3564 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3565 if (TLI.supportSwiftError()) {
3566 // Swifterror values can come from either a function parameter with
3567 // swifterror attribute or an alloca with swifterror attribute.
3568 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3569 if (Arg->hasSwiftErrorAttr())
3570 return visitStoreToSwiftError(I);
3573 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3574 if (Alloca->isSwiftError())
3575 return visitStoreToSwiftError(I);
3579 SmallVector<EVT, 4> ValueVTs;
3580 SmallVector<uint64_t, 4> Offsets;
3581 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3582 SrcV->getType(), ValueVTs, &Offsets);
3583 unsigned NumValues = ValueVTs.size();
3587 // Get the lowered operands. Note that we do this after
3588 // checking if NumResults is zero, because with zero results
3589 // the operands won't have values in the map.
3590 SDValue Src = getValue(SrcV);
3591 SDValue Ptr = getValue(PtrV);
3593 SDValue Root = getRoot();
3594 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3595 SDLoc dl = getCurSDLoc();
3596 EVT PtrVT = Ptr.getValueType();
3597 unsigned Alignment = I.getAlignment();
3599 I.getAAMetadata(AAInfo);
3601 auto MMOFlags = MachineMemOperand::MONone;
3603 MMOFlags |= MachineMemOperand::MOVolatile;
3604 if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3605 MMOFlags |= MachineMemOperand::MONonTemporal;
3607 // An aggregate load cannot wrap around the address space, so offsets to its
3608 // parts don't wrap either.
3610 Flags.setNoUnsignedWrap(true);
3612 unsigned ChainI = 0;
3613 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3614 // See visitLoad comments.
3615 if (ChainI == MaxParallelChains) {
3616 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3617 makeArrayRef(Chains.data(), ChainI));
3621 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3622 DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3623 SDValue St = DAG.getStore(
3624 Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3625 MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3626 Chains[ChainI] = St;
3629 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3630 makeArrayRef(Chains.data(), ChainI));
3631 DAG.setRoot(StoreNode);
3634 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3635 bool IsCompressing) {
3636 SDLoc sdl = getCurSDLoc();
3638 auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3639 unsigned& Alignment) {
3640 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3641 Src0 = I.getArgOperand(0);
3642 Ptr = I.getArgOperand(1);
3643 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3644 Mask = I.getArgOperand(3);
3646 auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3647 unsigned& Alignment) {
3648 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3649 Src0 = I.getArgOperand(0);
3650 Ptr = I.getArgOperand(1);
3651 Mask = I.getArgOperand(2);
3655 Value *PtrOperand, *MaskOperand, *Src0Operand;
3658 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3660 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3662 SDValue Ptr = getValue(PtrOperand);
3663 SDValue Src0 = getValue(Src0Operand);
3664 SDValue Mask = getValue(MaskOperand);
3666 EVT VT = Src0.getValueType();
3668 Alignment = DAG.getEVTAlignment(VT);
3671 I.getAAMetadata(AAInfo);
3673 MachineMemOperand *MMO =
3674 DAG.getMachineFunction().
3675 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3676 MachineMemOperand::MOStore, VT.getStoreSize(),
3678 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3679 MMO, false /* Truncating */,
3681 DAG.setRoot(StoreNode);
3682 setValue(&I, StoreNode);
3685 // Get a uniform base for the Gather/Scatter intrinsic.
3686 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3687 // We try to represent it as a base pointer + vector of indices.
3688 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3689 // The first operand of the GEP may be a single pointer or a vector of pointers
3691 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3693 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3694 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3696 // When the first GEP operand is a single pointer - it is the uniform base we
3697 // are looking for. If first operand of the GEP is a splat vector - we
3698 // extract the spalt value and use it as a uniform base.
3699 // In all other cases the function returns 'false'.
3701 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3702 SelectionDAGBuilder* SDB) {
3704 SelectionDAG& DAG = SDB->DAG;
3705 LLVMContext &Context = *DAG.getContext();
3707 assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3708 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3709 if (!GEP || GEP->getNumOperands() > 2)
3712 const Value *GEPPtr = GEP->getPointerOperand();
3713 if (!GEPPtr->getType()->isVectorTy())
3715 else if (!(Ptr = getSplatValue(GEPPtr)))
3718 Value *IndexVal = GEP->getOperand(1);
3720 // The operands of the GEP may be defined in another basic block.
3721 // In this case we'll not find nodes for the operands.
3722 if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3725 Base = SDB->getValue(Ptr);
3726 Index = SDB->getValue(IndexVal);
3728 // Suppress sign extension.
3729 if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3730 if (SDB->findValue(Sext->getOperand(0))) {
3731 IndexVal = Sext->getOperand(0);
3732 Index = SDB->getValue(IndexVal);
3735 if (!Index.getValueType().isVector()) {
3736 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3737 EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3738 Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3743 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3744 SDLoc sdl = getCurSDLoc();
3746 // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3747 const Value *Ptr = I.getArgOperand(1);
3748 SDValue Src0 = getValue(I.getArgOperand(0));
3749 SDValue Mask = getValue(I.getArgOperand(3));
3750 EVT VT = Src0.getValueType();
3751 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3753 Alignment = DAG.getEVTAlignment(VT);
3754 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3757 I.getAAMetadata(AAInfo);
3761 const Value *BasePtr = Ptr;
3762 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3764 const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3765 MachineMemOperand *MMO = DAG.getMachineFunction().
3766 getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3767 MachineMemOperand::MOStore, VT.getStoreSize(),
3770 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3771 Index = getValue(Ptr);
3773 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3774 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3776 DAG.setRoot(Scatter);
3777 setValue(&I, Scatter);
3780 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
3781 SDLoc sdl = getCurSDLoc();
3783 auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3784 unsigned& Alignment) {
3785 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3786 Ptr = I.getArgOperand(0);
3787 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
3788 Mask = I.getArgOperand(2);
3789 Src0 = I.getArgOperand(3);
3791 auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3792 unsigned& Alignment) {
3793 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
3794 Ptr = I.getArgOperand(0);
3796 Mask = I.getArgOperand(1);
3797 Src0 = I.getArgOperand(2);
3800 Value *PtrOperand, *MaskOperand, *Src0Operand;
3803 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3805 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3807 SDValue Ptr = getValue(PtrOperand);
3808 SDValue Src0 = getValue(Src0Operand);
3809 SDValue Mask = getValue(MaskOperand);
3811 EVT VT = Src0.getValueType();
3813 Alignment = DAG.getEVTAlignment(VT);
3816 I.getAAMetadata(AAInfo);
3817 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3819 // Do not serialize masked loads of constant memory with anything.
3820 bool AddToChain = !AA->pointsToConstantMemory(MemoryLocation(
3821 PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
3822 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
3824 MachineMemOperand *MMO =
3825 DAG.getMachineFunction().
3826 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3827 MachineMemOperand::MOLoad, VT.getStoreSize(),
3828 Alignment, AAInfo, Ranges);
3830 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3831 ISD::NON_EXTLOAD, IsExpanding);
3833 SDValue OutChain = Load.getValue(1);
3834 DAG.setRoot(OutChain);
3839 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3840 SDLoc sdl = getCurSDLoc();
3842 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3843 const Value *Ptr = I.getArgOperand(0);
3844 SDValue Src0 = getValue(I.getArgOperand(3));
3845 SDValue Mask = getValue(I.getArgOperand(2));
3847 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3848 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3849 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3851 Alignment = DAG.getEVTAlignment(VT);
3854 I.getAAMetadata(AAInfo);
3855 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3857 SDValue Root = DAG.getRoot();
3860 const Value *BasePtr = Ptr;
3861 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3862 bool ConstantMemory = false;
3864 AA->pointsToConstantMemory(MemoryLocation(
3865 BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
3867 // Do not serialize (non-volatile) loads of constant memory with anything.
3868 Root = DAG.getEntryNode();
3869 ConstantMemory = true;
3872 MachineMemOperand *MMO =
3873 DAG.getMachineFunction().
3874 getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3875 MachineMemOperand::MOLoad, VT.getStoreSize(),
3876 Alignment, AAInfo, Ranges);
3879 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3880 Index = getValue(Ptr);
3882 SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3883 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3886 SDValue OutChain = Gather.getValue(1);
3887 if (!ConstantMemory)
3888 PendingLoads.push_back(OutChain);
3889 setValue(&I, Gather);
3892 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
3893 SDLoc dl = getCurSDLoc();
3894 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3895 AtomicOrdering FailureOrder = I.getFailureOrdering();
3896 SynchronizationScope Scope = I.getSynchScope();
3898 SDValue InChain = getRoot();
3900 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
3901 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
3902 SDValue L = DAG.getAtomicCmpSwap(
3903 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
3904 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
3905 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
3906 /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
3908 SDValue OutChain = L.getValue(2);
3911 DAG.setRoot(OutChain);
3914 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
3915 SDLoc dl = getCurSDLoc();
3917 switch (I.getOperation()) {
3918 default: llvm_unreachable("Unknown atomicrmw operation");
3919 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
3920 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
3921 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
3922 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
3923 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
3924 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
3925 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
3926 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
3927 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
3928 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
3929 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
3931 AtomicOrdering Order = I.getOrdering();
3932 SynchronizationScope Scope = I.getSynchScope();
3934 SDValue InChain = getRoot();
3937 DAG.getAtomic(NT, dl,
3938 getValue(I.getValOperand()).getSimpleValueType(),
3940 getValue(I.getPointerOperand()),
3941 getValue(I.getValOperand()),
3942 I.getPointerOperand(),
3943 /* Alignment=*/ 0, Order, Scope);
3945 SDValue OutChain = L.getValue(1);
3948 DAG.setRoot(OutChain);
3951 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
3952 SDLoc dl = getCurSDLoc();
3953 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3956 Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
3957 TLI.getFenceOperandTy(DAG.getDataLayout()));
3958 Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
3959 TLI.getFenceOperandTy(DAG.getDataLayout()));
3960 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
3963 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
3964 SDLoc dl = getCurSDLoc();
3965 AtomicOrdering Order = I.getOrdering();
3966 SynchronizationScope Scope = I.getSynchScope();
3968 SDValue InChain = getRoot();
3970 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3971 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3973 if (I.getAlignment() < VT.getSizeInBits() / 8)
3974 report_fatal_error("Cannot generate unaligned atomic load");
3976 MachineMemOperand *MMO =
3977 DAG.getMachineFunction().
3978 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3979 MachineMemOperand::MOVolatile |
3980 MachineMemOperand::MOLoad,
3982 I.getAlignment() ? I.getAlignment() :
3983 DAG.getEVTAlignment(VT),
3984 AAMDNodes(), nullptr, Scope, Order);
3986 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
3988 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3989 getValue(I.getPointerOperand()), MMO);
3991 SDValue OutChain = L.getValue(1);
3994 DAG.setRoot(OutChain);
3997 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
3998 SDLoc dl = getCurSDLoc();
4000 AtomicOrdering Order = I.getOrdering();
4001 SynchronizationScope Scope = I.getSynchScope();
4003 SDValue InChain = getRoot();
4005 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4007 TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4009 if (I.getAlignment() < VT.getSizeInBits() / 8)
4010 report_fatal_error("Cannot generate unaligned atomic store");
4013 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4015 getValue(I.getPointerOperand()),
4016 getValue(I.getValueOperand()),
4017 I.getPointerOperand(), I.getAlignment(),
4020 DAG.setRoot(OutChain);
4023 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4025 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4026 unsigned Intrinsic) {
4027 // Ignore the callsite's attributes. A specific call site may be marked with
4028 // readnone, but the lowering code will expect the chain based on the
4030 const Function *F = I.getCalledFunction();
4031 bool HasChain = !F->doesNotAccessMemory();
4032 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4034 // Build the operand list.
4035 SmallVector<SDValue, 8> Ops;
4036 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4038 // We don't need to serialize loads against other loads.
4039 Ops.push_back(DAG.getRoot());
4041 Ops.push_back(getRoot());
4045 // Info is set by getTgtMemInstrinsic
4046 TargetLowering::IntrinsicInfo Info;
4047 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4048 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
4050 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4051 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4052 Info.opc == ISD::INTRINSIC_W_CHAIN)
4053 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4054 TLI.getPointerTy(DAG.getDataLayout())));
4056 // Add all operands of the call to the operand list.
4057 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4058 SDValue Op = getValue(I.getArgOperand(i));
4062 SmallVector<EVT, 4> ValueVTs;
4063 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4066 ValueVTs.push_back(MVT::Other);
4068 SDVTList VTs = DAG.getVTList(ValueVTs);
4072 if (IsTgtIntrinsic) {
4073 // This is target intrinsic that touches memory
4074 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
4075 VTs, Ops, Info.memVT,
4076 MachinePointerInfo(Info.ptrVal, Info.offset),
4077 Info.align, Info.vol,
4078 Info.readMem, Info.writeMem, Info.size);
4079 } else if (!HasChain) {
4080 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4081 } else if (!I.getType()->isVoidTy()) {
4082 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4084 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4088 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4090 PendingLoads.push_back(Chain);
4095 if (!I.getType()->isVoidTy()) {
4096 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4097 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4098 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4100 Result = lowerRangeToAssertZExt(DAG, I, Result);
4102 setValue(&I, Result);
4106 /// GetSignificand - Get the significand and build it into a floating-point
4107 /// number with exponent of 1:
4109 /// Op = (Op & 0x007fffff) | 0x3f800000;
4111 /// where Op is the hexadecimal representation of floating point value.
4112 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4113 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4114 DAG.getConstant(0x007fffff, dl, MVT::i32));
4115 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4116 DAG.getConstant(0x3f800000, dl, MVT::i32));
4117 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4120 /// GetExponent - Get the exponent:
4122 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4124 /// where Op is the hexadecimal representation of floating point value.
4125 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4126 const TargetLowering &TLI, const SDLoc &dl) {
4127 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4128 DAG.getConstant(0x7f800000, dl, MVT::i32));
4129 SDValue t1 = DAG.getNode(
4130 ISD::SRL, dl, MVT::i32, t0,
4131 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4132 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4133 DAG.getConstant(127, dl, MVT::i32));
4134 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4137 /// getF32Constant - Get 32-bit floating point constant.
4138 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4140 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4144 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4145 SelectionDAG &DAG) {
4146 // TODO: What fast-math-flags should be set on the floating-point nodes?
4148 // IntegerPartOfX = ((int32_t)(t0);
4149 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4151 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4152 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4153 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4155 // IntegerPartOfX <<= 23;
4156 IntegerPartOfX = DAG.getNode(
4157 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4158 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4159 DAG.getDataLayout())));
4161 SDValue TwoToFractionalPartOfX;
4162 if (LimitFloatPrecision <= 6) {
4163 // For floating-point precision of 6:
4165 // TwoToFractionalPartOfX =
4167 // (0.735607626f + 0.252464424f * x) * x;
4169 // error 0.0144103317, which is 6 bits
4170 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4171 getF32Constant(DAG, 0x3e814304, dl));
4172 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4173 getF32Constant(DAG, 0x3f3c50c8, dl));
4174 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4175 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4176 getF32Constant(DAG, 0x3f7f5e7e, dl));
4177 } else if (LimitFloatPrecision <= 12) {
4178 // For floating-point precision of 12:
4180 // TwoToFractionalPartOfX =
4183 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4185 // error 0.000107046256, which is 13 to 14 bits
4186 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4187 getF32Constant(DAG, 0x3da235e3, dl));
4188 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4189 getF32Constant(DAG, 0x3e65b8f3, dl));
4190 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4191 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4192 getF32Constant(DAG, 0x3f324b07, dl));
4193 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4194 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4195 getF32Constant(DAG, 0x3f7ff8fd, dl));
4196 } else { // LimitFloatPrecision <= 18
4197 // For floating-point precision of 18:
4199 // TwoToFractionalPartOfX =
4203 // (0.554906021e-1f +
4204 // (0.961591928e-2f +
4205 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4206 // error 2.47208000*10^(-7), which is better than 18 bits
4207 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4208 getF32Constant(DAG, 0x3924b03e, dl));
4209 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4210 getF32Constant(DAG, 0x3ab24b87, dl));
4211 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4212 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4213 getF32Constant(DAG, 0x3c1d8c17, dl));
4214 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4215 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4216 getF32Constant(DAG, 0x3d634a1d, dl));
4217 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4218 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4219 getF32Constant(DAG, 0x3e75fe14, dl));
4220 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4221 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4222 getF32Constant(DAG, 0x3f317234, dl));
4223 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4224 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4225 getF32Constant(DAG, 0x3f800000, dl));
4228 // Add the exponent into the result in integer domain.
4229 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4230 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4231 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4234 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4235 /// limited-precision mode.
4236 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4237 const TargetLowering &TLI) {
4238 if (Op.getValueType() == MVT::f32 &&
4239 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4241 // Put the exponent in the right bit position for later addition to the
4244 // #define LOG2OFe 1.4426950f
4245 // t0 = Op * LOG2OFe
4247 // TODO: What fast-math-flags should be set here?
4248 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4249 getF32Constant(DAG, 0x3fb8aa3b, dl));
4250 return getLimitedPrecisionExp2(t0, dl, DAG);
4253 // No special expansion.
4254 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4257 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4258 /// limited-precision mode.
4259 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4260 const TargetLowering &TLI) {
4262 // TODO: What fast-math-flags should be set on the floating-point nodes?
4264 if (Op.getValueType() == MVT::f32 &&
4265 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4266 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4268 // Scale the exponent by log(2) [0.69314718f].
4269 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4270 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4271 getF32Constant(DAG, 0x3f317218, dl));
4273 // Get the significand and build it into a floating-point number with
4275 SDValue X = GetSignificand(DAG, Op1, dl);
4277 SDValue LogOfMantissa;
4278 if (LimitFloatPrecision <= 6) {
4279 // For floating-point precision of 6:
4283 // (1.4034025f - 0.23903021f * x) * x;
4285 // error 0.0034276066, which is better than 8 bits
4286 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4287 getF32Constant(DAG, 0xbe74c456, dl));
4288 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4289 getF32Constant(DAG, 0x3fb3a2b1, dl));
4290 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4291 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4292 getF32Constant(DAG, 0x3f949a29, dl));
4293 } else if (LimitFloatPrecision <= 12) {
4294 // For floating-point precision of 12:
4300 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4302 // error 0.000061011436, which is 14 bits
4303 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4304 getF32Constant(DAG, 0xbd67b6d6, dl));
4305 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4306 getF32Constant(DAG, 0x3ee4f4b8, dl));
4307 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4308 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4309 getF32Constant(DAG, 0x3fbc278b, dl));
4310 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4311 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4312 getF32Constant(DAG, 0x40348e95, dl));
4313 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4314 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4315 getF32Constant(DAG, 0x3fdef31a, dl));
4316 } else { // LimitFloatPrecision <= 18
4317 // For floating-point precision of 18:
4325 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4327 // error 0.0000023660568, which is better than 18 bits
4328 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4329 getF32Constant(DAG, 0xbc91e5ac, dl));
4330 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4331 getF32Constant(DAG, 0x3e4350aa, dl));
4332 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4333 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4334 getF32Constant(DAG, 0x3f60d3e3, dl));
4335 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4336 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4337 getF32Constant(DAG, 0x4011cdf0, dl));
4338 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4339 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4340 getF32Constant(DAG, 0x406cfd1c, dl));
4341 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4342 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4343 getF32Constant(DAG, 0x408797cb, dl));
4344 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4345 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4346 getF32Constant(DAG, 0x4006dcab, dl));
4349 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4352 // No special expansion.
4353 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4356 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4357 /// limited-precision mode.
4358 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4359 const TargetLowering &TLI) {
4361 // TODO: What fast-math-flags should be set on the floating-point nodes?
4363 if (Op.getValueType() == MVT::f32 &&
4364 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4365 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4367 // Get the exponent.
4368 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4370 // Get the significand and build it into a floating-point number with
4372 SDValue X = GetSignificand(DAG, Op1, dl);
4374 // Different possible minimax approximations of significand in
4375 // floating-point for various degrees of accuracy over [1,2].
4376 SDValue Log2ofMantissa;
4377 if (LimitFloatPrecision <= 6) {
4378 // For floating-point precision of 6:
4380 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4382 // error 0.0049451742, which is more than 7 bits
4383 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4384 getF32Constant(DAG, 0xbeb08fe0, dl));
4385 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4386 getF32Constant(DAG, 0x40019463, dl));
4387 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4388 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4389 getF32Constant(DAG, 0x3fd6633d, dl));
4390 } else if (LimitFloatPrecision <= 12) {
4391 // For floating-point precision of 12:
4397 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4399 // error 0.0000876136000, which is better than 13 bits
4400 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4401 getF32Constant(DAG, 0xbda7262e, dl));
4402 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4403 getF32Constant(DAG, 0x3f25280b, dl));
4404 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4405 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4406 getF32Constant(DAG, 0x4007b923, dl));
4407 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4408 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4409 getF32Constant(DAG, 0x40823e2f, dl));
4410 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4411 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4412 getF32Constant(DAG, 0x4020d29c, dl));
4413 } else { // LimitFloatPrecision <= 18
4414 // For floating-point precision of 18:
4423 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4425 // error 0.0000018516, which is better than 18 bits
4426 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4427 getF32Constant(DAG, 0xbcd2769e, dl));
4428 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4429 getF32Constant(DAG, 0x3e8ce0b9, dl));
4430 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4431 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4432 getF32Constant(DAG, 0x3fa22ae7, dl));
4433 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4434 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4435 getF32Constant(DAG, 0x40525723, dl));
4436 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4437 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4438 getF32Constant(DAG, 0x40aaf200, dl));
4439 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4440 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4441 getF32Constant(DAG, 0x40c39dad, dl));
4442 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4443 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4444 getF32Constant(DAG, 0x4042902c, dl));
4447 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4450 // No special expansion.
4451 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4454 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4455 /// limited-precision mode.
4456 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4457 const TargetLowering &TLI) {
4459 // TODO: What fast-math-flags should be set on the floating-point nodes?
4461 if (Op.getValueType() == MVT::f32 &&
4462 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4463 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4465 // Scale the exponent by log10(2) [0.30102999f].
4466 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4467 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4468 getF32Constant(DAG, 0x3e9a209a, dl));
4470 // Get the significand and build it into a floating-point number with
4472 SDValue X = GetSignificand(DAG, Op1, dl);
4474 SDValue Log10ofMantissa;
4475 if (LimitFloatPrecision <= 6) {
4476 // For floating-point precision of 6:
4478 // Log10ofMantissa =
4480 // (0.60948995f - 0.10380950f * x) * x;
4482 // error 0.0014886165, which is 6 bits
4483 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4484 getF32Constant(DAG, 0xbdd49a13, dl));
4485 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4486 getF32Constant(DAG, 0x3f1c0789, dl));
4487 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4488 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4489 getF32Constant(DAG, 0x3f011300, dl));
4490 } else if (LimitFloatPrecision <= 12) {
4491 // For floating-point precision of 12:
4493 // Log10ofMantissa =
4496 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4498 // error 0.00019228036, which is better than 12 bits
4499 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4500 getF32Constant(DAG, 0x3d431f31, dl));
4501 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4502 getF32Constant(DAG, 0x3ea21fb2, dl));
4503 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4504 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4505 getF32Constant(DAG, 0x3f6ae232, dl));
4506 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4507 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4508 getF32Constant(DAG, 0x3f25f7c3, dl));
4509 } else { // LimitFloatPrecision <= 18
4510 // For floating-point precision of 18:
4512 // Log10ofMantissa =
4517 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4519 // error 0.0000037995730, which is better than 18 bits
4520 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4521 getF32Constant(DAG, 0x3c5d51ce, dl));
4522 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4523 getF32Constant(DAG, 0x3e00685a, dl));
4524 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4525 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4526 getF32Constant(DAG, 0x3efb6798, dl));
4527 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4528 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4529 getF32Constant(DAG, 0x3f88d192, dl));
4530 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4531 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4532 getF32Constant(DAG, 0x3fc4316c, dl));
4533 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4534 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4535 getF32Constant(DAG, 0x3f57ce70, dl));
4538 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4541 // No special expansion.
4542 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4545 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4546 /// limited-precision mode.
4547 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4548 const TargetLowering &TLI) {
4549 if (Op.getValueType() == MVT::f32 &&
4550 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4551 return getLimitedPrecisionExp2(Op, dl, DAG);
4553 // No special expansion.
4554 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4557 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4558 /// limited-precision mode with x == 10.0f.
4559 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4560 SelectionDAG &DAG, const TargetLowering &TLI) {
4561 bool IsExp10 = false;
4562 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4563 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4564 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4566 IsExp10 = LHSC->isExactlyValue(Ten);
4570 // TODO: What fast-math-flags should be set on the FMUL node?
4572 // Put the exponent in the right bit position for later addition to the
4575 // #define LOG2OF10 3.3219281f
4576 // t0 = Op * LOG2OF10;
4577 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4578 getF32Constant(DAG, 0x40549a78, dl));
4579 return getLimitedPrecisionExp2(t0, dl, DAG);
4582 // No special expansion.
4583 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4587 /// ExpandPowI - Expand a llvm.powi intrinsic.
4588 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4589 SelectionDAG &DAG) {
4590 // If RHS is a constant, we can expand this out to a multiplication tree,
4591 // otherwise we end up lowering to a call to __powidf2 (for example). When
4592 // optimizing for size, we only want to do this if the expansion would produce
4593 // a small number of multiplies, otherwise we do the full expansion.
4594 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4595 // Get the exponent as a positive value.
4596 unsigned Val = RHSC->getSExtValue();
4597 if ((int)Val < 0) Val = -Val;
4599 // powi(x, 0) -> 1.0
4601 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4603 const Function *F = DAG.getMachineFunction().getFunction();
4604 if (!F->optForSize() ||
4605 // If optimizing for size, don't insert too many multiplies.
4606 // This inserts up to 5 multiplies.
4607 countPopulation(Val) + Log2_32(Val) < 7) {
4608 // We use the simple binary decomposition method to generate the multiply
4609 // sequence. There are more optimal ways to do this (for example,
4610 // powi(x,15) generates one more multiply than it should), but this has
4611 // the benefit of being both really simple and much better than a libcall.
4612 SDValue Res; // Logically starts equal to 1.0
4613 SDValue CurSquare = LHS;
4614 // TODO: Intrinsics should have fast-math-flags that propagate to these
4619 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4621 Res = CurSquare; // 1.0*CurSquare.
4624 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4625 CurSquare, CurSquare);
4629 // If the original was negative, invert the result, producing 1/(x*x*x).
4630 if (RHSC->getSExtValue() < 0)
4631 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4632 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4637 // Otherwise, expand to a libcall.
4638 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4641 // getUnderlyingArgReg - Find underlying register used for a truncated or
4642 // bitcasted argument.
4643 static unsigned getUnderlyingArgReg(const SDValue &N) {
4644 switch (N.getOpcode()) {
4645 case ISD::CopyFromReg:
4646 return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4648 case ISD::AssertZext:
4649 case ISD::AssertSext:
4651 return getUnderlyingArgReg(N.getOperand(0));
4657 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4658 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
4659 /// At the end of instruction selection, they will be inserted to the entry BB.
4660 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4661 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4662 DILocation *DL, int64_t Offset, bool IsDbgDeclare, const SDValue &N) {
4663 const Argument *Arg = dyn_cast<Argument>(V);
4667 MachineFunction &MF = DAG.getMachineFunction();
4668 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4670 // Ignore inlined function arguments here.
4672 // FIXME: Should we be checking DL->inlinedAt() to determine this?
4673 if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4676 bool IsIndirect = false;
4677 Optional<MachineOperand> Op;
4678 // Some arguments' frame index is recorded during argument lowering.
4679 if (int FI = FuncInfo.getArgumentFrameIndex(Arg))
4680 Op = MachineOperand::CreateFI(FI);
4682 if (!Op && N.getNode()) {
4683 unsigned Reg = getUnderlyingArgReg(N);
4684 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4685 MachineRegisterInfo &RegInfo = MF.getRegInfo();
4686 unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4691 Op = MachineOperand::CreateReg(Reg, false);
4692 IsIndirect = IsDbgDeclare;
4697 // Check if ValueMap has reg number.
4698 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4699 if (VMI != FuncInfo.ValueMap.end()) {
4700 Op = MachineOperand::CreateReg(VMI->second, false);
4701 IsIndirect = IsDbgDeclare;
4705 if (!Op && N.getNode())
4706 // Check if frame index is available.
4707 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4708 if (FrameIndexSDNode *FINode =
4709 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4710 Op = MachineOperand::CreateFI(FINode->getIndex());
4715 assert(Variable->isValidLocationForIntrinsic(DL) &&
4716 "Expected inlined-at fields to agree");
4718 FuncInfo.ArgDbgValues.push_back(
4719 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4720 Op->getReg(), Offset, Variable, Expr));
4722 FuncInfo.ArgDbgValues.push_back(
4723 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4726 .addMetadata(Variable)
4727 .addMetadata(Expr));
4732 /// Return the appropriate SDDbgValue based on N.
4733 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4734 DILocalVariable *Variable,
4735 DIExpression *Expr, int64_t Offset,
4737 unsigned DbgSDNodeOrder) {
4739 auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode());
4740 if (FISDN && Expr->startsWithDeref()) {
4741 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4742 // stack slot locations as such instead of as indirectly addressed
4744 ArrayRef<uint64_t> TrailingElements(Expr->elements_begin() + 1,
4745 Expr->elements_end());
4746 DIExpression *DerefedDIExpr =
4747 DIExpression::get(*DAG.getContext(), TrailingElements);
4748 int FI = FISDN->getIndex();
4749 SDV = DAG.getFrameIndexDbgValue(Variable, DerefedDIExpr, FI, 0, dl,
4752 SDV = DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false,
4753 Offset, dl, DbgSDNodeOrder);
4758 // VisualStudio defines setjmp as _setjmp
4759 #if defined(_MSC_VER) && defined(setjmp) && \
4760 !defined(setjmp_undefined_for_msvc)
4761 # pragma push_macro("setjmp")
4763 # define setjmp_undefined_for_msvc
4766 /// Lower the call to the specified intrinsic function. If we want to emit this
4767 /// as a call to a named external function, return the name. Otherwise, lower it
4768 /// and return null.
4770 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4771 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4772 SDLoc sdl = getCurSDLoc();
4773 DebugLoc dl = getCurDebugLoc();
4776 switch (Intrinsic) {
4778 // By default, turn this into a target intrinsic node.
4779 visitTargetIntrinsic(I, Intrinsic);
4781 case Intrinsic::vastart: visitVAStart(I); return nullptr;
4782 case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4783 case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4784 case Intrinsic::returnaddress:
4785 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
4786 TLI.getPointerTy(DAG.getDataLayout()),
4787 getValue(I.getArgOperand(0))));
4789 case Intrinsic::addressofreturnaddress:
4790 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
4791 TLI.getPointerTy(DAG.getDataLayout())));
4793 case Intrinsic::frameaddress:
4794 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
4795 TLI.getPointerTy(DAG.getDataLayout()),
4796 getValue(I.getArgOperand(0))));
4798 case Intrinsic::read_register: {
4799 Value *Reg = I.getArgOperand(0);
4800 SDValue Chain = getRoot();
4802 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4803 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4804 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4805 DAG.getVTList(VT, MVT::Other), Chain, RegName);
4807 DAG.setRoot(Res.getValue(1));
4810 case Intrinsic::write_register: {
4811 Value *Reg = I.getArgOperand(0);
4812 Value *RegValue = I.getArgOperand(1);
4813 SDValue Chain = getRoot();
4815 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4816 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
4817 RegName, getValue(RegValue)));
4820 case Intrinsic::setjmp:
4821 return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4822 case Intrinsic::longjmp:
4823 return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4824 case Intrinsic::memcpy: {
4825 SDValue Op1 = getValue(I.getArgOperand(0));
4826 SDValue Op2 = getValue(I.getArgOperand(1));
4827 SDValue Op3 = getValue(I.getArgOperand(2));
4828 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4830 Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4831 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4832 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4833 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4835 MachinePointerInfo(I.getArgOperand(0)),
4836 MachinePointerInfo(I.getArgOperand(1)));
4837 updateDAGForMaybeTailCall(MC);
4840 case Intrinsic::memset: {
4841 SDValue Op1 = getValue(I.getArgOperand(0));
4842 SDValue Op2 = getValue(I.getArgOperand(1));
4843 SDValue Op3 = getValue(I.getArgOperand(2));
4844 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4846 Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4847 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4848 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4849 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4850 isTC, MachinePointerInfo(I.getArgOperand(0)));
4851 updateDAGForMaybeTailCall(MS);
4854 case Intrinsic::memmove: {
4855 SDValue Op1 = getValue(I.getArgOperand(0));
4856 SDValue Op2 = getValue(I.getArgOperand(1));
4857 SDValue Op3 = getValue(I.getArgOperand(2));
4858 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4860 Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4861 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4862 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4863 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4864 isTC, MachinePointerInfo(I.getArgOperand(0)),
4865 MachinePointerInfo(I.getArgOperand(1)));
4866 updateDAGForMaybeTailCall(MM);
4869 case Intrinsic::memcpy_element_atomic: {
4870 SDValue Dst = getValue(I.getArgOperand(0));
4871 SDValue Src = getValue(I.getArgOperand(1));
4872 SDValue NumElements = getValue(I.getArgOperand(2));
4873 SDValue ElementSize = getValue(I.getArgOperand(3));
4875 // Emit a library call.
4876 TargetLowering::ArgListTy Args;
4877 TargetLowering::ArgListEntry Entry;
4878 Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
4880 Args.push_back(Entry);
4883 Args.push_back(Entry);
4885 Entry.Ty = I.getArgOperand(2)->getType();
4886 Entry.Node = NumElements;
4887 Args.push_back(Entry);
4889 Entry.Ty = Type::getInt32Ty(*DAG.getContext());
4890 Entry.Node = ElementSize;
4891 Args.push_back(Entry);
4893 uint64_t ElementSizeConstant =
4894 cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4895 RTLIB::Libcall LibraryCall =
4896 RTLIB::getMEMCPY_ELEMENT_ATOMIC(ElementSizeConstant);
4897 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
4898 report_fatal_error("Unsupported element size");
4900 TargetLowering::CallLoweringInfo CLI(DAG);
4901 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
4902 TLI.getLibcallCallingConv(LibraryCall),
4903 Type::getVoidTy(*DAG.getContext()),
4904 DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall),
4905 TLI.getPointerTy(DAG.getDataLayout())),
4908 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4909 DAG.setRoot(CallResult.second);
4912 case Intrinsic::dbg_declare: {
4913 const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
4914 DILocalVariable *Variable = DI.getVariable();
4915 DIExpression *Expression = DI.getExpression();
4916 const Value *Address = DI.getAddress();
4917 assert(Variable && "Missing variable");
4919 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4923 // Check if address has undef value.
4924 if (isa<UndefValue>(Address) ||
4925 (Address->use_empty() && !isa<Argument>(Address))) {
4926 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4930 SDValue &N = NodeMap[Address];
4931 if (!N.getNode() && isa<Argument>(Address))
4932 // Check unused arguments map.
4933 N = UnusedArgNodeMap[Address];
4936 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
4937 Address = BCI->getOperand(0);
4938 // Parameters are handled specially.
4939 bool isParameter = Variable->isParameter() || isa<Argument>(Address);
4940 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
4941 if (isParameter && FINode) {
4942 // Byval parameter. We have a frame index at this point.
4943 SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
4944 FINode->getIndex(), 0, dl, SDNodeOrder);
4945 } else if (isa<Argument>(Address)) {
4946 // Address is an argument, so try to emit its dbg value using
4947 // virtual register info from the FuncInfo.ValueMap.
4948 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, true, N);
4951 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
4952 true, 0, dl, SDNodeOrder);
4954 DAG.AddDbgValue(SDV, N.getNode(), isParameter);
4956 // If Address is an argument then try to emit its dbg value using
4957 // virtual register info from the FuncInfo.ValueMap.
4958 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, true,
4960 // If variable is pinned by a alloca in dominating bb then
4961 // use StaticAllocaMap.
4962 if (const AllocaInst *AI = dyn_cast<AllocaInst>(Address)) {
4963 if (AI->getParent() != DI.getParent()) {
4964 DenseMap<const AllocaInst*, int>::iterator SI =
4965 FuncInfo.StaticAllocaMap.find(AI);
4966 if (SI != FuncInfo.StaticAllocaMap.end()) {
4967 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, SI->second,
4968 0, dl, SDNodeOrder);
4969 DAG.AddDbgValue(SDV, nullptr, false);
4974 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
4979 case Intrinsic::dbg_value: {
4980 const DbgValueInst &DI = cast<DbgValueInst>(I);
4981 assert(DI.getVariable() && "Missing variable");
4983 DILocalVariable *Variable = DI.getVariable();
4984 DIExpression *Expression = DI.getExpression();
4985 uint64_t Offset = DI.getOffset();
4986 const Value *V = DI.getValue();
4991 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
4992 SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
4994 DAG.AddDbgValue(SDV, nullptr, false);
4996 // Do not use getValue() in here; we don't want to generate code at
4997 // this point if it hasn't been done yet.
4998 SDValue N = NodeMap[V];
4999 if (!N.getNode() && isa<Argument>(V))
5000 // Check unused arguments map.
5001 N = UnusedArgNodeMap[V];
5003 if (!EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset,
5005 SDV = getDbgValue(N, Variable, Expression, Offset, dl, SDNodeOrder);
5006 DAG.AddDbgValue(SDV, N.getNode(), false);
5008 } else if (!V->use_empty() ) {
5009 // Do not call getValue(V) yet, as we don't want to generate code.
5010 // Remember it for later.
5011 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
5012 DanglingDebugInfoMap[V] = DDI;
5014 // We may expand this to cover more cases. One case where we have no
5015 // data available is an unreferenced parameter.
5016 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5020 // Build a debug info table entry.
5021 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(V))
5022 V = BCI->getOperand(0);
5023 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
5024 // Don't handle byval struct arguments or VLAs, for example.
5026 DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n");
5027 DEBUG(dbgs() << " Last seen at:\n " << *V << "\n");
5030 DenseMap<const AllocaInst*, int>::iterator SI =
5031 FuncInfo.StaticAllocaMap.find(AI);
5032 if (SI == FuncInfo.StaticAllocaMap.end())
5033 return nullptr; // VLAs.
5037 case Intrinsic::eh_typeid_for: {
5038 // Find the type id for the given typeinfo.
5039 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5040 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5041 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5046 case Intrinsic::eh_return_i32:
5047 case Intrinsic::eh_return_i64:
5048 DAG.getMachineFunction().setCallsEHReturn(true);
5049 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5052 getValue(I.getArgOperand(0)),
5053 getValue(I.getArgOperand(1))));
5055 case Intrinsic::eh_unwind_init:
5056 DAG.getMachineFunction().setCallsUnwindInit(true);
5058 case Intrinsic::eh_dwarf_cfa: {
5059 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5060 TLI.getPointerTy(DAG.getDataLayout()),
5061 getValue(I.getArgOperand(0))));
5064 case Intrinsic::eh_sjlj_callsite: {
5065 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5066 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5067 assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
5068 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
5070 MMI.setCurrentCallSite(CI->getZExtValue());
5073 case Intrinsic::eh_sjlj_functioncontext: {
5074 // Get and store the index of the function context.
5075 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5077 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5078 int FI = FuncInfo.StaticAllocaMap[FnCtx];
5079 MFI.setFunctionContextIndex(FI);
5082 case Intrinsic::eh_sjlj_setjmp: {
5085 Ops[1] = getValue(I.getArgOperand(0));
5086 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5087 DAG.getVTList(MVT::i32, MVT::Other), Ops);
5088 setValue(&I, Op.getValue(0));
5089 DAG.setRoot(Op.getValue(1));
5092 case Intrinsic::eh_sjlj_longjmp: {
5093 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5094 getRoot(), getValue(I.getArgOperand(0))));
5097 case Intrinsic::eh_sjlj_setup_dispatch: {
5098 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5103 case Intrinsic::masked_gather:
5104 visitMaskedGather(I);
5106 case Intrinsic::masked_load:
5109 case Intrinsic::masked_scatter:
5110 visitMaskedScatter(I);
5112 case Intrinsic::masked_store:
5113 visitMaskedStore(I);
5115 case Intrinsic::masked_expandload:
5116 visitMaskedLoad(I, true /* IsExpanding */);
5118 case Intrinsic::masked_compressstore:
5119 visitMaskedStore(I, true /* IsCompressing */);
5121 case Intrinsic::x86_mmx_pslli_w:
5122 case Intrinsic::x86_mmx_pslli_d:
5123 case Intrinsic::x86_mmx_pslli_q:
5124 case Intrinsic::x86_mmx_psrli_w:
5125 case Intrinsic::x86_mmx_psrli_d:
5126 case Intrinsic::x86_mmx_psrli_q:
5127 case Intrinsic::x86_mmx_psrai_w:
5128 case Intrinsic::x86_mmx_psrai_d: {
5129 SDValue ShAmt = getValue(I.getArgOperand(1));
5130 if (isa<ConstantSDNode>(ShAmt)) {
5131 visitTargetIntrinsic(I, Intrinsic);
5134 unsigned NewIntrinsic = 0;
5135 EVT ShAmtVT = MVT::v2i32;
5136 switch (Intrinsic) {
5137 case Intrinsic::x86_mmx_pslli_w:
5138 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5140 case Intrinsic::x86_mmx_pslli_d:
5141 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5143 case Intrinsic::x86_mmx_pslli_q:
5144 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5146 case Intrinsic::x86_mmx_psrli_w:
5147 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5149 case Intrinsic::x86_mmx_psrli_d:
5150 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5152 case Intrinsic::x86_mmx_psrli_q:
5153 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5155 case Intrinsic::x86_mmx_psrai_w:
5156 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5158 case Intrinsic::x86_mmx_psrai_d:
5159 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5161 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5164 // The vector shift intrinsics with scalars uses 32b shift amounts but
5165 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5167 // We must do this early because v2i32 is not a legal type.
5170 ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5171 ShAmt = DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5172 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5173 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5174 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5175 DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5176 getValue(I.getArgOperand(0)), ShAmt);
5180 case Intrinsic::powi:
5181 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5182 getValue(I.getArgOperand(1)), DAG));
5184 case Intrinsic::log:
5185 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5187 case Intrinsic::log2:
5188 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5190 case Intrinsic::log10:
5191 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5193 case Intrinsic::exp:
5194 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5196 case Intrinsic::exp2:
5197 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5199 case Intrinsic::pow:
5200 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5201 getValue(I.getArgOperand(1)), DAG, TLI));
5203 case Intrinsic::sqrt:
5204 case Intrinsic::fabs:
5205 case Intrinsic::sin:
5206 case Intrinsic::cos:
5207 case Intrinsic::floor:
5208 case Intrinsic::ceil:
5209 case Intrinsic::trunc:
5210 case Intrinsic::rint:
5211 case Intrinsic::nearbyint:
5212 case Intrinsic::round:
5213 case Intrinsic::canonicalize: {
5215 switch (Intrinsic) {
5216 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5217 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5218 case Intrinsic::fabs: Opcode = ISD::FABS; break;
5219 case Intrinsic::sin: Opcode = ISD::FSIN; break;
5220 case Intrinsic::cos: Opcode = ISD::FCOS; break;
5221 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5222 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5223 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5224 case Intrinsic::rint: Opcode = ISD::FRINT; break;
5225 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5226 case Intrinsic::round: Opcode = ISD::FROUND; break;
5227 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5230 setValue(&I, DAG.getNode(Opcode, sdl,
5231 getValue(I.getArgOperand(0)).getValueType(),
5232 getValue(I.getArgOperand(0))));
5235 case Intrinsic::minnum: {
5236 auto VT = getValue(I.getArgOperand(0)).getValueType();
5238 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT)
5241 setValue(&I, DAG.getNode(Opc, sdl, VT,
5242 getValue(I.getArgOperand(0)),
5243 getValue(I.getArgOperand(1))));
5246 case Intrinsic::maxnum: {
5247 auto VT = getValue(I.getArgOperand(0)).getValueType();
5249 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT)
5252 setValue(&I, DAG.getNode(Opc, sdl, VT,
5253 getValue(I.getArgOperand(0)),
5254 getValue(I.getArgOperand(1))));
5257 case Intrinsic::copysign:
5258 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5259 getValue(I.getArgOperand(0)).getValueType(),
5260 getValue(I.getArgOperand(0)),
5261 getValue(I.getArgOperand(1))));
5263 case Intrinsic::fma:
5264 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5265 getValue(I.getArgOperand(0)).getValueType(),
5266 getValue(I.getArgOperand(0)),
5267 getValue(I.getArgOperand(1)),
5268 getValue(I.getArgOperand(2))));
5270 case Intrinsic::experimental_constrained_fadd:
5271 case Intrinsic::experimental_constrained_fsub:
5272 case Intrinsic::experimental_constrained_fmul:
5273 case Intrinsic::experimental_constrained_fdiv:
5274 case Intrinsic::experimental_constrained_frem:
5275 visitConstrainedFPIntrinsic(I, Intrinsic);
5277 case Intrinsic::fmuladd: {
5278 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5279 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5280 TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5281 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5282 getValue(I.getArgOperand(0)).getValueType(),
5283 getValue(I.getArgOperand(0)),
5284 getValue(I.getArgOperand(1)),
5285 getValue(I.getArgOperand(2))));
5287 // TODO: Intrinsic calls should have fast-math-flags.
5288 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5289 getValue(I.getArgOperand(0)).getValueType(),
5290 getValue(I.getArgOperand(0)),
5291 getValue(I.getArgOperand(1)));
5292 SDValue Add = DAG.getNode(ISD::FADD, sdl,
5293 getValue(I.getArgOperand(0)).getValueType(),
5295 getValue(I.getArgOperand(2)));
5300 case Intrinsic::convert_to_fp16:
5301 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5302 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5303 getValue(I.getArgOperand(0)),
5304 DAG.getTargetConstant(0, sdl,
5307 case Intrinsic::convert_from_fp16:
5308 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5309 TLI.getValueType(DAG.getDataLayout(), I.getType()),
5310 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5311 getValue(I.getArgOperand(0)))));
5313 case Intrinsic::pcmarker: {
5314 SDValue Tmp = getValue(I.getArgOperand(0));
5315 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5318 case Intrinsic::readcyclecounter: {
5319 SDValue Op = getRoot();
5320 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5321 DAG.getVTList(MVT::i64, MVT::Other), Op);
5323 DAG.setRoot(Res.getValue(1));
5326 case Intrinsic::bitreverse:
5327 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5328 getValue(I.getArgOperand(0)).getValueType(),
5329 getValue(I.getArgOperand(0))));
5331 case Intrinsic::bswap:
5332 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5333 getValue(I.getArgOperand(0)).getValueType(),
5334 getValue(I.getArgOperand(0))));
5336 case Intrinsic::cttz: {
5337 SDValue Arg = getValue(I.getArgOperand(0));
5338 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5339 EVT Ty = Arg.getValueType();
5340 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5344 case Intrinsic::ctlz: {
5345 SDValue Arg = getValue(I.getArgOperand(0));
5346 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5347 EVT Ty = Arg.getValueType();
5348 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5352 case Intrinsic::ctpop: {
5353 SDValue Arg = getValue(I.getArgOperand(0));
5354 EVT Ty = Arg.getValueType();
5355 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5358 case Intrinsic::stacksave: {
5359 SDValue Op = getRoot();
5361 ISD::STACKSAVE, sdl,
5362 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5364 DAG.setRoot(Res.getValue(1));
5367 case Intrinsic::stackrestore: {
5368 Res = getValue(I.getArgOperand(0));
5369 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5372 case Intrinsic::get_dynamic_area_offset: {
5373 SDValue Op = getRoot();
5374 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5375 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5376 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5379 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5381 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5387 case Intrinsic::stackguard: {
5388 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5389 MachineFunction &MF = DAG.getMachineFunction();
5390 const Module &M = *MF.getFunction()->getParent();
5391 SDValue Chain = getRoot();
5392 if (TLI.useLoadStackGuardNode()) {
5393 Res = getLoadStackGuard(DAG, sdl, Chain);
5395 const Value *Global = TLI.getSDagStackGuard(M);
5396 unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5397 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5398 MachinePointerInfo(Global, 0), Align,
5399 MachineMemOperand::MOVolatile);
5405 case Intrinsic::stackprotector: {
5406 // Emit code into the DAG to store the stack guard onto the stack.
5407 MachineFunction &MF = DAG.getMachineFunction();
5408 MachineFrameInfo &MFI = MF.getFrameInfo();
5409 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5410 SDValue Src, Chain = getRoot();
5412 if (TLI.useLoadStackGuardNode())
5413 Src = getLoadStackGuard(DAG, sdl, Chain);
5415 Src = getValue(I.getArgOperand(0)); // The guard's value.
5417 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5419 int FI = FuncInfo.StaticAllocaMap[Slot];
5420 MFI.setStackProtectorIndex(FI);
5422 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5424 // Store the stack protector onto the stack.
5425 Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5426 DAG.getMachineFunction(), FI),
5427 /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5432 case Intrinsic::objectsize: {
5433 // If we don't know by now, we're never going to know.
5434 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5436 assert(CI && "Non-constant type in __builtin_object_size?");
5438 SDValue Arg = getValue(I.getCalledValue());
5439 EVT Ty = Arg.getValueType();
5442 Res = DAG.getConstant(-1ULL, sdl, Ty);
5444 Res = DAG.getConstant(0, sdl, Ty);
5449 case Intrinsic::annotation:
5450 case Intrinsic::ptr_annotation:
5451 case Intrinsic::invariant_group_barrier:
5452 // Drop the intrinsic, but forward the value
5453 setValue(&I, getValue(I.getOperand(0)));
5455 case Intrinsic::assume:
5456 case Intrinsic::var_annotation:
5457 // Discard annotate attributes and assumptions
5460 case Intrinsic::init_trampoline: {
5461 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5465 Ops[1] = getValue(I.getArgOperand(0));
5466 Ops[2] = getValue(I.getArgOperand(1));
5467 Ops[3] = getValue(I.getArgOperand(2));
5468 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5469 Ops[5] = DAG.getSrcValue(F);
5471 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5476 case Intrinsic::adjust_trampoline: {
5477 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5478 TLI.getPointerTy(DAG.getDataLayout()),
5479 getValue(I.getArgOperand(0))));
5482 case Intrinsic::gcroot: {
5483 MachineFunction &MF = DAG.getMachineFunction();
5484 const Function *F = MF.getFunction();
5486 assert(F->hasGC() &&
5487 "only valid in functions with gc specified, enforced by Verifier");
5488 assert(GFI && "implied by previous");
5489 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5490 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5492 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5493 GFI->addStackRoot(FI->getIndex(), TypeMap);
5496 case Intrinsic::gcread:
5497 case Intrinsic::gcwrite:
5498 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
5499 case Intrinsic::flt_rounds:
5500 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5503 case Intrinsic::expect: {
5504 // Just replace __builtin_expect(exp, c) with EXP.
5505 setValue(&I, getValue(I.getArgOperand(0)));
5509 case Intrinsic::debugtrap:
5510 case Intrinsic::trap: {
5511 StringRef TrapFuncName =
5513 .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
5514 .getValueAsString();
5515 if (TrapFuncName.empty()) {
5516 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5517 ISD::TRAP : ISD::DEBUGTRAP;
5518 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5521 TargetLowering::ArgListTy Args;
5523 TargetLowering::CallLoweringInfo CLI(DAG);
5524 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5525 CallingConv::C, I.getType(),
5526 DAG.getExternalSymbol(TrapFuncName.data(),
5527 TLI.getPointerTy(DAG.getDataLayout())),
5530 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5531 DAG.setRoot(Result.second);
5535 case Intrinsic::uadd_with_overflow:
5536 case Intrinsic::sadd_with_overflow:
5537 case Intrinsic::usub_with_overflow:
5538 case Intrinsic::ssub_with_overflow:
5539 case Intrinsic::umul_with_overflow:
5540 case Intrinsic::smul_with_overflow: {
5542 switch (Intrinsic) {
5543 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5544 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5545 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5546 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5547 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5548 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5549 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5551 SDValue Op1 = getValue(I.getArgOperand(0));
5552 SDValue Op2 = getValue(I.getArgOperand(1));
5554 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5555 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5558 case Intrinsic::prefetch: {
5560 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5562 Ops[1] = getValue(I.getArgOperand(0));
5563 Ops[2] = getValue(I.getArgOperand(1));
5564 Ops[3] = getValue(I.getArgOperand(2));
5565 Ops[4] = getValue(I.getArgOperand(3));
5566 DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
5567 DAG.getVTList(MVT::Other), Ops,
5568 EVT::getIntegerVT(*Context, 8),
5569 MachinePointerInfo(I.getArgOperand(0)),
5571 false, /* volatile */
5573 rw==1)); /* write */
5576 case Intrinsic::lifetime_start:
5577 case Intrinsic::lifetime_end: {
5578 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
5579 // Stack coloring is not enabled in O0, discard region information.
5580 if (TM.getOptLevel() == CodeGenOpt::None)
5583 SmallVector<Value *, 4> Allocas;
5584 GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
5586 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
5587 E = Allocas.end(); Object != E; ++Object) {
5588 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
5590 // Could not find an Alloca.
5591 if (!LifetimeObject)
5594 // First check that the Alloca is static, otherwise it won't have a
5595 // valid frame index.
5596 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
5597 if (SI == FuncInfo.StaticAllocaMap.end())
5600 int FI = SI->second;
5605 DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true);
5606 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
5608 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
5613 case Intrinsic::invariant_start:
5614 // Discard region information.
5615 setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
5617 case Intrinsic::invariant_end:
5618 // Discard region information.
5620 case Intrinsic::clear_cache:
5621 return TLI.getClearCacheBuiltinName();
5622 case Intrinsic::donothing:
5625 case Intrinsic::experimental_stackmap: {
5629 case Intrinsic::experimental_patchpoint_void:
5630 case Intrinsic::experimental_patchpoint_i64: {
5631 visitPatchpoint(&I);
5634 case Intrinsic::experimental_gc_statepoint: {
5635 LowerStatepoint(ImmutableStatepoint(&I));
5638 case Intrinsic::experimental_gc_result: {
5639 visitGCResult(cast<GCResultInst>(I));
5642 case Intrinsic::experimental_gc_relocate: {
5643 visitGCRelocate(cast<GCRelocateInst>(I));
5646 case Intrinsic::instrprof_increment:
5647 llvm_unreachable("instrprof failed to lower an increment");
5648 case Intrinsic::instrprof_value_profile:
5649 llvm_unreachable("instrprof failed to lower a value profiling call");
5650 case Intrinsic::localescape: {
5651 MachineFunction &MF = DAG.getMachineFunction();
5652 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5654 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
5655 // is the same on all targets.
5656 for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
5657 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
5658 if (isa<ConstantPointerNull>(Arg))
5659 continue; // Skip null pointers. They represent a hole in index space.
5660 AllocaInst *Slot = cast<AllocaInst>(Arg);
5661 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
5662 "can only escape static allocas");
5663 int FI = FuncInfo.StaticAllocaMap[Slot];
5664 MCSymbol *FrameAllocSym =
5665 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5666 GlobalValue::getRealLinkageName(MF.getName()), Idx);
5667 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
5668 TII->get(TargetOpcode::LOCAL_ESCAPE))
5669 .addSym(FrameAllocSym)
5676 case Intrinsic::localrecover: {
5677 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
5678 MachineFunction &MF = DAG.getMachineFunction();
5679 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
5681 // Get the symbol that defines the frame offset.
5682 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
5683 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
5684 unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX));
5685 MCSymbol *FrameAllocSym =
5686 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5687 GlobalValue::getRealLinkageName(Fn->getName()), IdxVal);
5689 // Create a MCSymbol for the label to avoid any target lowering
5690 // that would make this PC relative.
5691 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
5693 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
5695 // Add the offset to the FP.
5696 Value *FP = I.getArgOperand(1);
5697 SDValue FPVal = getValue(FP);
5698 SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
5704 case Intrinsic::eh_exceptionpointer:
5705 case Intrinsic::eh_exceptioncode: {
5706 // Get the exception pointer vreg, copy from it, and resize it to fit.
5707 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
5708 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5709 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
5710 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
5712 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
5713 if (Intrinsic == Intrinsic::eh_exceptioncode)
5714 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
5719 case Intrinsic::experimental_deoptimize:
5720 LowerDeoptimizeCall(&I);
5725 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(const CallInst &I,
5726 unsigned Intrinsic) {
5727 SDLoc sdl = getCurSDLoc();
5729 switch (Intrinsic) {
5730 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5731 case Intrinsic::experimental_constrained_fadd:
5732 Opcode = ISD::STRICT_FADD;
5734 case Intrinsic::experimental_constrained_fsub:
5735 Opcode = ISD::STRICT_FSUB;
5737 case Intrinsic::experimental_constrained_fmul:
5738 Opcode = ISD::STRICT_FMUL;
5740 case Intrinsic::experimental_constrained_fdiv:
5741 Opcode = ISD::STRICT_FDIV;
5743 case Intrinsic::experimental_constrained_frem:
5744 Opcode = ISD::STRICT_FREM;
5747 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5748 SDValue Chain = getRoot();
5749 SDValue Ops[3] = { Chain, getValue(I.getArgOperand(0)),
5750 getValue(I.getArgOperand(1)) };
5751 SmallVector<EVT, 4> ValueVTs;
5752 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5753 ValueVTs.push_back(MVT::Other); // Out chain
5755 SDVTList VTs = DAG.getVTList(ValueVTs);
5756 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Ops);
5758 assert(Result.getNode()->getNumValues() == 2);
5759 SDValue OutChain = Result.getValue(1);
5760 DAG.setRoot(OutChain);
5761 SDValue FPResult = Result.getValue(0);
5762 setValue(&I, FPResult);
5765 std::pair<SDValue, SDValue>
5766 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
5767 const BasicBlock *EHPadBB) {
5768 MachineFunction &MF = DAG.getMachineFunction();
5769 MachineModuleInfo &MMI = MF.getMMI();
5770 MCSymbol *BeginLabel = nullptr;
5773 // Insert a label before the invoke call to mark the try range. This can be
5774 // used to detect deletion of the invoke via the MachineModuleInfo.
5775 BeginLabel = MMI.getContext().createTempSymbol();
5777 // For SjLj, keep track of which landing pads go with which invokes
5778 // so as to maintain the ordering of pads in the LSDA.
5779 unsigned CallSiteIndex = MMI.getCurrentCallSite();
5780 if (CallSiteIndex) {
5781 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
5782 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
5784 // Now that the call site is handled, stop tracking it.
5785 MMI.setCurrentCallSite(0);
5788 // Both PendingLoads and PendingExports must be flushed here;
5789 // this call might not return.
5791 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
5793 CLI.setChain(getRoot());
5795 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5796 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5798 assert((CLI.IsTailCall || Result.second.getNode()) &&
5799 "Non-null chain expected with non-tail call!");
5800 assert((Result.second.getNode() || !Result.first.getNode()) &&
5801 "Null value expected with tail call!");
5803 if (!Result.second.getNode()) {
5804 // As a special case, a null chain means that a tail call has been emitted
5805 // and the DAG root is already updated.
5808 // Since there's no actual continuation from this block, nothing can be
5809 // relying on us setting vregs for them.
5810 PendingExports.clear();
5812 DAG.setRoot(Result.second);
5816 // Insert a label at the end of the invoke call to mark the try range. This
5817 // can be used to detect deletion of the invoke via the MachineModuleInfo.
5818 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
5819 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
5821 // Inform MachineModuleInfo of range.
5822 if (MF.hasEHFunclets()) {
5824 WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
5825 EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS->getInstruction()),
5826 BeginLabel, EndLabel);
5828 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
5835 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
5837 const BasicBlock *EHPadBB) {
5838 auto &DL = DAG.getDataLayout();
5839 FunctionType *FTy = CS.getFunctionType();
5840 Type *RetTy = CS.getType();
5842 TargetLowering::ArgListTy Args;
5843 Args.reserve(CS.arg_size());
5845 const Value *SwiftErrorVal = nullptr;
5846 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5848 // We can't tail call inside a function with a swifterror argument. Lowering
5849 // does not support this yet. It would have to move into the swifterror
5850 // register before the call.
5851 auto *Caller = CS.getInstruction()->getParent()->getParent();
5852 if (TLI.supportSwiftError() &&
5853 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
5856 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
5858 TargetLowering::ArgListEntry Entry;
5859 const Value *V = *i;
5862 if (V->getType()->isEmptyTy())
5865 SDValue ArgNode = getValue(V);
5866 Entry.Node = ArgNode; Entry.Ty = V->getType();
5868 Entry.setAttributes(&CS, i - CS.arg_begin());
5870 // Use swifterror virtual register as input to the call.
5871 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
5873 // We find the virtual register for the actual swifterror argument.
5874 // Instead of using the Value, we use the virtual register instead.
5876 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, V),
5877 EVT(TLI.getPointerTy(DL)));
5880 Args.push_back(Entry);
5882 // If we have an explicit sret argument that is an Instruction, (i.e., it
5883 // might point to function-local memory), we can't meaningfully tail-call.
5884 if (Entry.IsSRet && isa<Instruction>(V))
5888 // Check if target-independent constraints permit a tail call here.
5889 // Target-dependent constraints are checked within TLI->LowerCallTo.
5890 if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
5893 // Disable tail calls if there is an swifterror argument. Targets have not
5894 // been updated to support tail calls.
5895 if (TLI.supportSwiftError() && SwiftErrorVal)
5898 TargetLowering::CallLoweringInfo CLI(DAG);
5899 CLI.setDebugLoc(getCurSDLoc())
5900 .setChain(getRoot())
5901 .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
5902 .setTailCall(isTailCall)
5903 .setConvergent(CS.isConvergent());
5904 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
5906 if (Result.first.getNode()) {
5907 const Instruction *Inst = CS.getInstruction();
5908 Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
5909 setValue(Inst, Result.first);
5912 // The last element of CLI.InVals has the SDValue for swifterror return.
5913 // Here we copy it to a virtual register and update SwiftErrorMap for
5915 if (SwiftErrorVal && TLI.supportSwiftError()) {
5916 // Get the last element of InVals.
5917 SDValue Src = CLI.InVals.back();
5918 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
5919 unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
5920 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
5921 // We update the virtual register for the actual swifterror argument.
5922 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
5923 DAG.setRoot(CopyNode);
5927 /// Return true if it only matters that the value is equal or not-equal to zero.
5928 static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
5929 for (const User *U : V->users()) {
5930 if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
5931 if (IC->isEquality())
5932 if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
5933 if (C->isNullValue())
5935 // Unknown instruction.
5941 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
5942 SelectionDAGBuilder &Builder) {
5944 // Check to see if this load can be trivially constant folded, e.g. if the
5945 // input is from a string literal.
5946 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
5947 // Cast pointer to the type we really want to load.
5949 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
5950 if (LoadVT.isVector())
5951 LoadTy = VectorType::get(LoadTy, LoadVT.getVectorNumElements());
5953 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
5954 PointerType::getUnqual(LoadTy));
5956 if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
5957 const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
5958 return Builder.getValue(LoadCst);
5961 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
5962 // still constant memory, the input chain can be the entry node.
5964 bool ConstantMemory = false;
5966 // Do not serialize (non-volatile) loads of constant memory with anything.
5967 if (Builder.AA->pointsToConstantMemory(PtrVal)) {
5968 Root = Builder.DAG.getEntryNode();
5969 ConstantMemory = true;
5971 // Do not serialize non-volatile loads against each other.
5972 Root = Builder.DAG.getRoot();
5975 SDValue Ptr = Builder.getValue(PtrVal);
5976 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
5977 Ptr, MachinePointerInfo(PtrVal),
5978 /* Alignment = */ 1);
5980 if (!ConstantMemory)
5981 Builder.PendingLoads.push_back(LoadVal.getValue(1));
5985 /// Record the value for an instruction that produces an integer result,
5986 /// converting the type where necessary.
5987 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
5990 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
5993 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
5995 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
5996 setValue(&I, Value);
5999 /// See if we can lower a memcmp call into an optimized form. If so, return
6000 /// true and lower it. Otherwise return false, and it will be lowered like a
6002 /// The caller already checked that \p I calls the appropriate LibFunc with a
6003 /// correct prototype.
6004 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6005 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6006 const Value *Size = I.getArgOperand(2);
6007 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6008 if (CSize && CSize->getZExtValue() == 0) {
6009 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6011 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6015 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6016 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
6017 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
6018 getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
6019 if (Res.first.getNode()) {
6020 processIntegerCallValue(I, Res.first, true);
6021 PendingLoads.push_back(Res.second);
6025 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
6026 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
6027 if (!CSize || !IsOnlyUsedInZeroEqualityComparison(&I))
6030 // If the target has a fast compare for the given size, it will return a
6031 // preferred load type for that size. Require that the load VT is legal and
6032 // that the target supports unaligned loads of that type. Otherwise, return
6034 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
6035 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6036 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
6037 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
6038 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6039 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6040 // TODO: Check alignment of src and dest ptrs.
6041 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6042 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6043 if (!TLI.isTypeLegal(LVT) ||
6044 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
6045 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
6046 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
6052 // This turns into unaligned loads. We only do this if the target natively
6053 // supports the MVT we'll be loading or if it is small enough (<= 4) that
6054 // we'll only produce a small number of byte loads.
6056 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
6057 switch (NumBitsToCompare) {
6069 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
6073 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
6076 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
6077 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
6079 // Bitcast to a wide integer type if the loads are vectors.
6080 if (LoadVT.isVector()) {
6081 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
6082 LoadL = DAG.getBitcast(CmpVT, LoadL);
6083 LoadR = DAG.getBitcast(CmpVT, LoadR);
6086 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
6087 processIntegerCallValue(I, Cmp, false);
6091 /// See if we can lower a memchr call into an optimized form. If so, return
6092 /// true and lower it. Otherwise return false, and it will be lowered like a
6094 /// The caller already checked that \p I calls the appropriate LibFunc with a
6095 /// correct prototype.
6096 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6097 const Value *Src = I.getArgOperand(0);
6098 const Value *Char = I.getArgOperand(1);
6099 const Value *Length = I.getArgOperand(2);
6101 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6102 std::pair<SDValue, SDValue> Res =
6103 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6104 getValue(Src), getValue(Char), getValue(Length),
6105 MachinePointerInfo(Src));
6106 if (Res.first.getNode()) {
6107 setValue(&I, Res.first);
6108 PendingLoads.push_back(Res.second);
6115 /// See if we can lower a mempcpy call into an optimized form. If so, return
6116 /// true and lower it. Otherwise return false, and it will be lowered like a
6118 /// The caller already checked that \p I calls the appropriate LibFunc with a
6119 /// correct prototype.
6120 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6121 SDValue Dst = getValue(I.getArgOperand(0));
6122 SDValue Src = getValue(I.getArgOperand(1));
6123 SDValue Size = getValue(I.getArgOperand(2));
6125 unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6126 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6127 unsigned Align = std::min(DstAlign, SrcAlign);
6128 if (Align == 0) // Alignment of one or both could not be inferred.
6129 Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6132 SDLoc sdl = getCurSDLoc();
6134 // In the mempcpy context we need to pass in a false value for isTailCall
6135 // because the return pointer needs to be adjusted by the size of
6136 // the copied memory.
6137 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6138 false, /*isTailCall=*/false,
6139 MachinePointerInfo(I.getArgOperand(0)),
6140 MachinePointerInfo(I.getArgOperand(1)));
6141 assert(MC.getNode() != nullptr &&
6142 "** memcpy should not be lowered as TailCall in mempcpy context **");
6145 // Check if Size needs to be truncated or extended.
6146 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6148 // Adjust return pointer to point just past the last dst byte.
6149 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6151 setValue(&I, DstPlusSize);
6155 /// See if we can lower a strcpy call into an optimized form. If so, return
6156 /// true and lower it, otherwise return false and it will be lowered like a
6158 /// The caller already checked that \p I calls the appropriate LibFunc with a
6159 /// correct prototype.
6160 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6161 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6163 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6164 std::pair<SDValue, SDValue> Res =
6165 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6166 getValue(Arg0), getValue(Arg1),
6167 MachinePointerInfo(Arg0),
6168 MachinePointerInfo(Arg1), isStpcpy);
6169 if (Res.first.getNode()) {
6170 setValue(&I, Res.first);
6171 DAG.setRoot(Res.second);
6178 /// See if we can lower a strcmp call into an optimized form. If so, return
6179 /// true and lower it, otherwise return false and it will be lowered like a
6181 /// The caller already checked that \p I calls the appropriate LibFunc with a
6182 /// correct prototype.
6183 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6184 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6186 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6187 std::pair<SDValue, SDValue> Res =
6188 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6189 getValue(Arg0), getValue(Arg1),
6190 MachinePointerInfo(Arg0),
6191 MachinePointerInfo(Arg1));
6192 if (Res.first.getNode()) {
6193 processIntegerCallValue(I, Res.first, true);
6194 PendingLoads.push_back(Res.second);
6201 /// See if we can lower a strlen call into an optimized form. If so, return
6202 /// true and lower it, otherwise return false and it will be lowered like a
6204 /// The caller already checked that \p I calls the appropriate LibFunc with a
6205 /// correct prototype.
6206 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6207 const Value *Arg0 = I.getArgOperand(0);
6209 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6210 std::pair<SDValue, SDValue> Res =
6211 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6212 getValue(Arg0), MachinePointerInfo(Arg0));
6213 if (Res.first.getNode()) {
6214 processIntegerCallValue(I, Res.first, false);
6215 PendingLoads.push_back(Res.second);
6222 /// See if we can lower a strnlen call into an optimized form. If so, return
6223 /// true and lower it, otherwise return false and it will be lowered like a
6225 /// The caller already checked that \p I calls the appropriate LibFunc with a
6226 /// correct prototype.
6227 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6228 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6230 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6231 std::pair<SDValue, SDValue> Res =
6232 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6233 getValue(Arg0), getValue(Arg1),
6234 MachinePointerInfo(Arg0));
6235 if (Res.first.getNode()) {
6236 processIntegerCallValue(I, Res.first, false);
6237 PendingLoads.push_back(Res.second);
6244 /// See if we can lower a unary floating-point operation into an SDNode with
6245 /// the specified Opcode. If so, return true and lower it, otherwise return
6246 /// false and it will be lowered like a normal call.
6247 /// The caller already checked that \p I calls the appropriate LibFunc with a
6248 /// correct prototype.
6249 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6251 // We already checked this call's prototype; verify it doesn't modify errno.
6252 if (!I.onlyReadsMemory())
6255 SDValue Tmp = getValue(I.getArgOperand(0));
6256 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6260 /// See if we can lower a binary floating-point operation into an SDNode with
6261 /// the specified Opcode. If so, return true and lower it. Otherwise return
6262 /// false, and it will be lowered like a normal call.
6263 /// The caller already checked that \p I calls the appropriate LibFunc with a
6264 /// correct prototype.
6265 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6267 // We already checked this call's prototype; verify it doesn't modify errno.
6268 if (!I.onlyReadsMemory())
6271 SDValue Tmp0 = getValue(I.getArgOperand(0));
6272 SDValue Tmp1 = getValue(I.getArgOperand(1));
6273 EVT VT = Tmp0.getValueType();
6274 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6278 void SelectionDAGBuilder::visitCall(const CallInst &I) {
6279 // Handle inline assembly differently.
6280 if (isa<InlineAsm>(I.getCalledValue())) {
6285 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6286 computeUsesVAFloatArgument(I, MMI);
6288 const char *RenameFn = nullptr;
6289 if (Function *F = I.getCalledFunction()) {
6290 if (F->isDeclaration()) {
6291 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
6292 if (unsigned IID = II->getIntrinsicID(F)) {
6293 RenameFn = visitIntrinsicCall(I, IID);
6298 if (Intrinsic::ID IID = F->getIntrinsicID()) {
6299 RenameFn = visitIntrinsicCall(I, IID);
6305 // Check for well-known libc/libm calls. If the function is internal, it
6306 // can't be a library call. Don't do the check if marked as nobuiltin for
6309 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
6310 LibInfo->getLibFunc(*F, Func) &&
6311 LibInfo->hasOptimizedCodeGen(Func)) {
6314 case LibFunc_copysign:
6315 case LibFunc_copysignf:
6316 case LibFunc_copysignl:
6317 // We already checked this call's prototype; verify it doesn't modify
6319 if (I.onlyReadsMemory()) {
6320 SDValue LHS = getValue(I.getArgOperand(0));
6321 SDValue RHS = getValue(I.getArgOperand(1));
6322 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
6323 LHS.getValueType(), LHS, RHS));
6330 if (visitUnaryFloatCall(I, ISD::FABS))
6336 if (visitBinaryFloatCall(I, ISD::FMINNUM))
6342 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
6348 if (visitUnaryFloatCall(I, ISD::FSIN))
6354 if (visitUnaryFloatCall(I, ISD::FCOS))
6360 case LibFunc_sqrt_finite:
6361 case LibFunc_sqrtf_finite:
6362 case LibFunc_sqrtl_finite:
6363 if (visitUnaryFloatCall(I, ISD::FSQRT))
6367 case LibFunc_floorf:
6368 case LibFunc_floorl:
6369 if (visitUnaryFloatCall(I, ISD::FFLOOR))
6372 case LibFunc_nearbyint:
6373 case LibFunc_nearbyintf:
6374 case LibFunc_nearbyintl:
6375 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
6381 if (visitUnaryFloatCall(I, ISD::FCEIL))
6387 if (visitUnaryFloatCall(I, ISD::FRINT))
6391 case LibFunc_roundf:
6392 case LibFunc_roundl:
6393 if (visitUnaryFloatCall(I, ISD::FROUND))
6397 case LibFunc_truncf:
6398 case LibFunc_truncl:
6399 if (visitUnaryFloatCall(I, ISD::FTRUNC))
6405 if (visitUnaryFloatCall(I, ISD::FLOG2))
6411 if (visitUnaryFloatCall(I, ISD::FEXP2))
6414 case LibFunc_memcmp:
6415 if (visitMemCmpCall(I))
6418 case LibFunc_mempcpy:
6419 if (visitMemPCpyCall(I))
6422 case LibFunc_memchr:
6423 if (visitMemChrCall(I))
6426 case LibFunc_strcpy:
6427 if (visitStrCpyCall(I, false))
6430 case LibFunc_stpcpy:
6431 if (visitStrCpyCall(I, true))
6434 case LibFunc_strcmp:
6435 if (visitStrCmpCall(I))
6438 case LibFunc_strlen:
6439 if (visitStrLenCall(I))
6442 case LibFunc_strnlen:
6443 if (visitStrNLenCall(I))
6452 Callee = getValue(I.getCalledValue());
6454 Callee = DAG.getExternalSymbol(
6456 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
6458 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
6459 // have to do anything here to lower funclet bundles.
6460 assert(!I.hasOperandBundlesOtherThan(
6461 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
6462 "Cannot lower calls with arbitrary operand bundles!");
6464 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
6465 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
6467 // Check if we can potentially perform a tail call. More detailed checking
6468 // is be done within LowerCallTo, after more information about the call is
6470 LowerCallTo(&I, Callee, I.isTailCall());
6475 /// AsmOperandInfo - This contains information for each constraint that we are
6477 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
6479 /// CallOperand - If this is the result output operand or a clobber
6480 /// this is null, otherwise it is the incoming operand to the CallInst.
6481 /// This gets modified as the asm is processed.
6482 SDValue CallOperand;
6484 /// AssignedRegs - If this is a register or register class operand, this
6485 /// contains the set of register corresponding to the operand.
6486 RegsForValue AssignedRegs;
6488 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
6489 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
6492 /// Whether or not this operand accesses memory
6493 bool hasMemory(const TargetLowering &TLI) const {
6494 // Indirect operand accesses access memory.
6498 for (const auto &Code : Codes)
6499 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
6505 /// getCallOperandValEVT - Return the EVT of the Value* that this operand
6506 /// corresponds to. If there is no Value* for this operand, it returns
6508 EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
6509 const DataLayout &DL) const {
6510 if (!CallOperandVal) return MVT::Other;
6512 if (isa<BasicBlock>(CallOperandVal))
6513 return TLI.getPointerTy(DL);
6515 llvm::Type *OpTy = CallOperandVal->getType();
6517 // FIXME: code duplicated from TargetLowering::ParseConstraints().
6518 // If this is an indirect operand, the operand is a pointer to the
6521 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
6523 report_fatal_error("Indirect operand for inline asm not a pointer!");
6524 OpTy = PtrTy->getElementType();
6527 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6528 if (StructType *STy = dyn_cast<StructType>(OpTy))
6529 if (STy->getNumElements() == 1)
6530 OpTy = STy->getElementType(0);
6532 // If OpTy is not a single value, it may be a struct/union that we
6533 // can tile with integers.
6534 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6535 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
6544 OpTy = IntegerType::get(Context, BitSize);
6549 return TLI.getValueType(DL, OpTy, true);
6553 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
6555 } // end anonymous namespace
6557 /// Make sure that the output operand \p OpInfo and its corresponding input
6558 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
6560 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
6561 SDISelAsmOperandInfo &MatchingOpInfo,
6562 SelectionDAG &DAG) {
6563 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
6566 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
6567 const auto &TLI = DAG.getTargetLoweringInfo();
6569 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6570 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
6571 OpInfo.ConstraintVT);
6572 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6573 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
6574 MatchingOpInfo.ConstraintVT);
6575 if ((OpInfo.ConstraintVT.isInteger() !=
6576 MatchingOpInfo.ConstraintVT.isInteger()) ||
6577 (MatchRC.second != InputRC.second)) {
6578 // FIXME: error out in a more elegant fashion
6579 report_fatal_error("Unsupported asm: input constraint"
6580 " with a matching output constraint of"
6581 " incompatible type!");
6583 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
6586 /// Get a direct memory input to behave well as an indirect operand.
6587 /// This may introduce stores, hence the need for a \p Chain.
6588 /// \return The (possibly updated) chain.
6589 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
6590 SDISelAsmOperandInfo &OpInfo,
6591 SelectionDAG &DAG) {
6592 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6594 // If we don't have an indirect input, put it in the constpool if we can,
6595 // otherwise spill it to a stack slot.
6596 // TODO: This isn't quite right. We need to handle these according to
6597 // the addressing mode that the constraint wants. Also, this may take
6598 // an additional register for the computation and we don't want that
6601 // If the operand is a float, integer, or vector constant, spill to a
6602 // constant pool entry to get its address.
6603 const Value *OpVal = OpInfo.CallOperandVal;
6604 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
6605 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
6606 OpInfo.CallOperand = DAG.getConstantPool(
6607 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
6611 // Otherwise, create a stack slot and emit a store to it before the asm.
6612 Type *Ty = OpVal->getType();
6613 auto &DL = DAG.getDataLayout();
6614 uint64_t TySize = DL.getTypeAllocSize(Ty);
6615 unsigned Align = DL.getPrefTypeAlignment(Ty);
6616 MachineFunction &MF = DAG.getMachineFunction();
6617 int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
6618 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
6619 Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
6620 MachinePointerInfo::getFixedStack(MF, SSFI));
6621 OpInfo.CallOperand = StackSlot;
6626 /// GetRegistersForValue - Assign registers (virtual or physical) for the
6627 /// specified operand. We prefer to assign virtual registers, to allow the
6628 /// register allocator to handle the assignment process. However, if the asm
6629 /// uses features that we can't model on machineinstrs, we have SDISel do the
6630 /// allocation. This produces generally horrible, but correct, code.
6632 /// OpInfo describes the operand.
6634 static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
6636 SDISelAsmOperandInfo &OpInfo) {
6637 LLVMContext &Context = *DAG.getContext();
6639 MachineFunction &MF = DAG.getMachineFunction();
6640 SmallVector<unsigned, 4> Regs;
6641 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
6643 // If this is a constraint for a single physreg, or a constraint for a
6644 // register class, find it.
6645 std::pair<unsigned, const TargetRegisterClass *> PhysReg =
6646 TLI.getRegForInlineAsmConstraint(&TRI, OpInfo.ConstraintCode,
6647 OpInfo.ConstraintVT);
6649 unsigned NumRegs = 1;
6650 if (OpInfo.ConstraintVT != MVT::Other) {
6651 // If this is a FP input in an integer register (or visa versa) insert a bit
6652 // cast of the input value. More generally, handle any case where the input
6653 // value disagrees with the register class we plan to stick this in.
6654 if (OpInfo.Type == InlineAsm::isInput && PhysReg.second &&
6655 !TRI.isTypeLegalForClass(*PhysReg.second, OpInfo.ConstraintVT)) {
6656 // Try to convert to the first EVT that the reg class contains. If the
6657 // types are identical size, use a bitcast to convert (e.g. two differing
6659 MVT RegVT = *TRI.legalclasstypes_begin(*PhysReg.second);
6660 if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
6661 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6662 RegVT, OpInfo.CallOperand);
6663 OpInfo.ConstraintVT = RegVT;
6664 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
6665 // If the input is a FP value and we want it in FP registers, do a
6666 // bitcast to the corresponding integer type. This turns an f64 value
6667 // into i64, which can be passed with two i32 values on a 32-bit
6669 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
6670 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6671 RegVT, OpInfo.CallOperand);
6672 OpInfo.ConstraintVT = RegVT;
6676 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
6680 EVT ValueVT = OpInfo.ConstraintVT;
6682 // If this is a constraint for a specific physical register, like {r17},
6684 if (unsigned AssignedReg = PhysReg.first) {
6685 const TargetRegisterClass *RC = PhysReg.second;
6686 if (OpInfo.ConstraintVT == MVT::Other)
6687 ValueVT = *TRI.legalclasstypes_begin(*RC);
6689 // Get the actual register value type. This is important, because the user
6690 // may have asked for (e.g.) the AX register in i32 type. We need to
6691 // remember that AX is actually i16 to get the right extension.
6692 RegVT = *TRI.legalclasstypes_begin(*RC);
6694 // This is a explicit reference to a physical register.
6695 Regs.push_back(AssignedReg);
6697 // If this is an expanded reference, add the rest of the regs to Regs.
6699 TargetRegisterClass::iterator I = RC->begin();
6700 for (; *I != AssignedReg; ++I)
6701 assert(I != RC->end() && "Didn't find reg!");
6703 // Already added the first reg.
6705 for (; NumRegs; --NumRegs, ++I) {
6706 assert(I != RC->end() && "Ran out of registers to allocate!");
6711 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6715 // Otherwise, if this was a reference to an LLVM register class, create vregs
6716 // for this reference.
6717 if (const TargetRegisterClass *RC = PhysReg.second) {
6718 RegVT = *TRI.legalclasstypes_begin(*RC);
6719 if (OpInfo.ConstraintVT == MVT::Other)
6722 // Create the appropriate number of virtual registers.
6723 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6724 for (; NumRegs; --NumRegs)
6725 Regs.push_back(RegInfo.createVirtualRegister(RC));
6727 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6731 // Otherwise, we couldn't allocate enough registers for this.
6735 findMatchingInlineAsmOperand(unsigned OperandNo,
6736 const std::vector<SDValue> &AsmNodeOperands) {
6737 // Scan until we find the definition we already emitted of this operand.
6738 unsigned CurOp = InlineAsm::Op_FirstOperand;
6739 for (; OperandNo; --OperandNo) {
6740 // Advance to the next operand.
6742 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6743 assert((InlineAsm::isRegDefKind(OpFlag) ||
6744 InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
6745 InlineAsm::isMemKind(OpFlag)) &&
6746 "Skipped past definitions?");
6747 CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
6752 /// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
6753 /// \return true if it has succeeded, false otherwise
6754 static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
6755 MVT RegVT, SelectionDAG &DAG) {
6756 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6757 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
6758 for (unsigned i = 0, e = NumRegs; i != e; ++i) {
6759 if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
6760 Regs.push_back(RegInfo.createVirtualRegister(RC));
6771 explicit ExtraFlags(ImmutableCallSite CS) {
6772 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6773 if (IA->hasSideEffects())
6774 Flags |= InlineAsm::Extra_HasSideEffects;
6775 if (IA->isAlignStack())
6776 Flags |= InlineAsm::Extra_IsAlignStack;
6777 if (CS.isConvergent())
6778 Flags |= InlineAsm::Extra_IsConvergent;
6779 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
6782 void update(const llvm::TargetLowering::AsmOperandInfo &OpInfo) {
6783 // Ideally, we would only check against memory constraints. However, the
6784 // meaning of an Other constraint can be target-specific and we can't easily
6785 // reason about it. Therefore, be conservative and set MayLoad/MayStore
6786 // for Other constraints as well.
6787 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
6788 OpInfo.ConstraintType == TargetLowering::C_Other) {
6789 if (OpInfo.Type == InlineAsm::isInput)
6790 Flags |= InlineAsm::Extra_MayLoad;
6791 else if (OpInfo.Type == InlineAsm::isOutput)
6792 Flags |= InlineAsm::Extra_MayStore;
6793 else if (OpInfo.Type == InlineAsm::isClobber)
6794 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
6798 unsigned get() const { return Flags; }
6801 /// visitInlineAsm - Handle a call to an InlineAsm object.
6803 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
6804 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6806 /// ConstraintOperands - Information about all of the constraints.
6807 SDISelAsmOperandInfoVector ConstraintOperands;
6809 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6810 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
6811 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
6813 bool hasMemory = false;
6815 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6816 ExtraFlags ExtraInfo(CS);
6818 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6819 unsigned ResNo = 0; // ResNo - The result number of the next output.
6820 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
6821 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
6822 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
6824 MVT OpVT = MVT::Other;
6826 // Compute the value type for each operand.
6827 if (OpInfo.Type == InlineAsm::isInput ||
6828 (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
6829 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
6831 // Process the call argument. BasicBlocks are labels, currently appearing
6833 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
6834 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
6836 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
6841 .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
6845 if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
6846 // The return value of the call is this value. As such, there is no
6847 // corresponding argument.
6848 assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
6849 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
6850 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
6851 STy->getElementType(ResNo));
6853 assert(ResNo == 0 && "Asm only has one result!");
6854 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
6859 OpInfo.ConstraintVT = OpVT;
6862 hasMemory = OpInfo.hasMemory(TLI);
6864 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
6865 // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
6866 auto TargetConstraint = TargetConstraints[i];
6868 // Compute the constraint code and ConstraintType to use.
6869 TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
6871 ExtraInfo.update(TargetConstraint);
6874 SDValue Chain, Flag;
6876 // We won't need to flush pending loads if this asm doesn't touch
6877 // memory and is nonvolatile.
6878 if (hasMemory || IA->hasSideEffects())
6881 Chain = DAG.getRoot();
6883 // Second pass over the constraints: compute which constraint option to use
6884 // and assign registers to constraints that want a specific physreg.
6885 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6886 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6888 // If this is an output operand with a matching input operand, look up the
6889 // matching input. If their types mismatch, e.g. one is an integer, the
6890 // other is floating point, or their sizes are different, flag it as an
6892 if (OpInfo.hasMatchingInput()) {
6893 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6894 patchMatchingInput(OpInfo, Input, DAG);
6897 // Compute the constraint code and ConstraintType to use.
6898 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
6900 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6901 OpInfo.Type == InlineAsm::isClobber)
6904 // If this is a memory input, and if the operand is not indirect, do what we
6905 // need to to provide an address for the memory input.
6906 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6907 !OpInfo.isIndirect) {
6908 assert((OpInfo.isMultipleAlternative ||
6909 (OpInfo.Type == InlineAsm::isInput)) &&
6910 "Can only indirectify direct input operands!");
6912 // Memory operands really want the address of the value.
6913 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
6915 // There is no longer a Value* corresponding to this operand.
6916 OpInfo.CallOperandVal = nullptr;
6918 // It is now an indirect operand.
6919 OpInfo.isIndirect = true;
6922 // If this constraint is for a specific register, allocate it before
6924 if (OpInfo.ConstraintType == TargetLowering::C_Register)
6925 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6928 // Third pass - Loop over all of the operands, assigning virtual or physregs
6929 // to register class operands.
6930 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6931 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6933 // C_Register operands have already been allocated, Other/Memory don't need
6935 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
6936 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6939 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
6940 std::vector<SDValue> AsmNodeOperands;
6941 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
6942 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
6943 IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
6945 // If we have a !srcloc metadata node associated with it, we want to attach
6946 // this to the ultimately generated inline asm machineinstr. To do this, we
6947 // pass in the third operand as this (potentially null) inline asm MDNode.
6948 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
6949 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
6951 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6952 // bits as operand 3.
6953 AsmNodeOperands.push_back(DAG.getTargetConstant(
6954 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
6956 // Loop over all of the inputs, copying the operand values into the
6957 // appropriate registers and processing the output regs.
6958 RegsForValue RetValRegs;
6960 // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
6961 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
6963 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6964 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6966 switch (OpInfo.Type) {
6967 case InlineAsm::isOutput: {
6968 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
6969 OpInfo.ConstraintType != TargetLowering::C_Register) {
6970 // Memory output, or 'other' output (e.g. 'X' constraint).
6971 assert(OpInfo.isIndirect && "Memory output must be indirect operand");
6973 unsigned ConstraintID =
6974 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
6975 assert(ConstraintID != InlineAsm::Constraint_Unknown &&
6976 "Failed to convert memory constraint code to constraint id.");
6978 // Add information to the INLINEASM node to know about this output.
6979 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
6980 OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
6981 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
6983 AsmNodeOperands.push_back(OpInfo.CallOperand);
6987 // Otherwise, this is a register or register class output.
6989 // Copy the output from the appropriate register. Find a register that
6991 if (OpInfo.AssignedRegs.Regs.empty()) {
6993 CS, "couldn't allocate output register for constraint '" +
6994 Twine(OpInfo.ConstraintCode) + "'");
6998 // If this is an indirect operand, store through the pointer after the
7000 if (OpInfo.isIndirect) {
7001 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7002 OpInfo.CallOperandVal));
7004 // This is the result value of the call.
7005 assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7006 // Concatenate this output onto the outputs list.
7007 RetValRegs.append(OpInfo.AssignedRegs);
7010 // Add information to the INLINEASM node to know that this register is
7013 .AddInlineAsmOperands(OpInfo.isEarlyClobber
7014 ? InlineAsm::Kind_RegDefEarlyClobber
7015 : InlineAsm::Kind_RegDef,
7016 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7019 case InlineAsm::isInput: {
7020 SDValue InOperandVal = OpInfo.CallOperand;
7022 if (OpInfo.isMatchingInputConstraint()) {
7023 // If this is required to match an output register we have already set,
7024 // just use its register.
7025 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7028 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7029 if (InlineAsm::isRegDefKind(OpFlag) ||
7030 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7031 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7032 if (OpInfo.isIndirect) {
7033 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7034 emitInlineAsmError(CS, "inline asm not supported yet:"
7035 " don't know how to handle tied "
7036 "indirect register inputs");
7040 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7041 SmallVector<unsigned, 4> Regs;
7043 if (!createVirtualRegs(Regs,
7044 InlineAsm::getNumOperandRegisters(OpFlag),
7046 emitInlineAsmError(CS, "inline asm error: This value type register "
7047 "class is not natively supported!");
7051 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7053 SDLoc dl = getCurSDLoc();
7054 // Use the produced MatchedRegs object to
7055 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7056 Chain, &Flag, CS.getInstruction());
7057 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7058 true, OpInfo.getMatchedOperand(), dl,
7059 DAG, AsmNodeOperands);
7063 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
7064 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
7065 "Unexpected number of operands");
7066 // Add information to the INLINEASM node to know about this input.
7067 // See InlineAsm.h isUseOperandTiedToDef.
7068 OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7069 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7070 OpInfo.getMatchedOperand());
7071 AsmNodeOperands.push_back(DAG.getTargetConstant(
7072 OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7073 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7077 // Treat indirect 'X' constraint as memory.
7078 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7080 OpInfo.ConstraintType = TargetLowering::C_Memory;
7082 if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7083 std::vector<SDValue> Ops;
7084 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7087 emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7088 Twine(OpInfo.ConstraintCode) + "'");
7092 // Add information to the INLINEASM node to know about this input.
7093 unsigned ResOpType =
7094 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7095 AsmNodeOperands.push_back(DAG.getTargetConstant(
7096 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7097 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7101 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7102 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
7103 assert(InOperandVal.getValueType() ==
7104 TLI.getPointerTy(DAG.getDataLayout()) &&
7105 "Memory operands expect pointer values");
7107 unsigned ConstraintID =
7108 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7109 assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7110 "Failed to convert memory constraint code to constraint id.");
7112 // Add information to the INLINEASM node to know about this input.
7113 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7114 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7115 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7118 AsmNodeOperands.push_back(InOperandVal);
7122 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
7123 OpInfo.ConstraintType == TargetLowering::C_Register) &&
7124 "Unknown constraint type!");
7126 // TODO: Support this.
7127 if (OpInfo.isIndirect) {
7129 CS, "Don't know how to handle indirect register inputs yet "
7130 "for constraint '" +
7131 Twine(OpInfo.ConstraintCode) + "'");
7135 // Copy the input into the appropriate registers.
7136 if (OpInfo.AssignedRegs.Regs.empty()) {
7137 emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7138 Twine(OpInfo.ConstraintCode) + "'");
7142 SDLoc dl = getCurSDLoc();
7144 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7145 Chain, &Flag, CS.getInstruction());
7147 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7148 dl, DAG, AsmNodeOperands);
7151 case InlineAsm::isClobber: {
7152 // Add the clobbered value to the operand list, so that the register
7153 // allocator is aware that the physreg got clobbered.
7154 if (!OpInfo.AssignedRegs.Regs.empty())
7155 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7156 false, 0, getCurSDLoc(), DAG,
7163 // Finish up input operands. Set the input chain and add the flag last.
7164 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7165 if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7167 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7168 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7169 Flag = Chain.getValue(1);
7171 // If this asm returns a register value, copy the result from that register
7172 // and set it as the value of the call.
7173 if (!RetValRegs.Regs.empty()) {
7174 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7175 Chain, &Flag, CS.getInstruction());
7177 // FIXME: Why don't we do this for inline asms with MRVs?
7178 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
7179 EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7181 // If any of the results of the inline asm is a vector, it may have the
7182 // wrong width/num elts. This can happen for register classes that can
7183 // contain multiple different value types. The preg or vreg allocated may
7184 // not have the same VT as was expected. Convert it to the right type
7185 // with bit_convert.
7186 if (ResultType != Val.getValueType() && Val.getValueType().isVector()) {
7187 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(),
7190 } else if (ResultType != Val.getValueType() &&
7191 ResultType.isInteger() && Val.getValueType().isInteger()) {
7192 // If a result value was tied to an input value, the computed result may
7193 // have a wider width than the expected result. Extract the relevant
7195 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
7198 assert(ResultType == Val.getValueType() && "Asm result value mismatch!");
7201 setValue(CS.getInstruction(), Val);
7202 // Don't need to use this as a chain in this case.
7203 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7207 std::vector<std::pair<SDValue, const Value *> > StoresToEmit;
7209 // Process indirect outputs, first output all of the flagged copies out of
7211 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7212 RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7213 const Value *Ptr = IndirectStoresToEmit[i].second;
7214 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7216 StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7219 // Emit the non-flagged stores from the physregs.
7220 SmallVector<SDValue, 8> OutChains;
7221 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7222 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7223 getValue(StoresToEmit[i].second),
7224 MachinePointerInfo(StoresToEmit[i].second));
7225 OutChains.push_back(Val);
7228 if (!OutChains.empty())
7229 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7234 void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7235 const Twine &Message) {
7236 LLVMContext &Ctx = *DAG.getContext();
7237 Ctx.emitError(CS.getInstruction(), Message);
7239 // Make sure we leave the DAG in a valid state
7240 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7241 auto VT = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7242 setValue(CS.getInstruction(), DAG.getUNDEF(VT));
7245 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7246 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
7247 MVT::Other, getRoot(),
7248 getValue(I.getArgOperand(0)),
7249 DAG.getSrcValue(I.getArgOperand(0))));
7252 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7253 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7254 const DataLayout &DL = DAG.getDataLayout();
7255 SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7256 getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7257 DAG.getSrcValue(I.getOperand(0)),
7258 DL.getABITypeAlignment(I.getType()));
7260 DAG.setRoot(V.getValue(1));
7263 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
7264 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
7265 MVT::Other, getRoot(),
7266 getValue(I.getArgOperand(0)),
7267 DAG.getSrcValue(I.getArgOperand(0))));
7270 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
7271 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
7272 MVT::Other, getRoot(),
7273 getValue(I.getArgOperand(0)),
7274 getValue(I.getArgOperand(1)),
7275 DAG.getSrcValue(I.getArgOperand(0)),
7276 DAG.getSrcValue(I.getArgOperand(1))));
7279 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
7280 const Instruction &I,
7282 const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
7286 ConstantRange CR = getConstantRangeFromMetadata(*Range);
7287 if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
7290 APInt Lo = CR.getUnsignedMin();
7291 if (!Lo.isMinValue())
7294 APInt Hi = CR.getUnsignedMax();
7295 unsigned Bits = Hi.getActiveBits();
7297 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
7299 SDLoc SL = getCurSDLoc();
7301 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
7302 DAG.getValueType(SmallVT));
7303 unsigned NumVals = Op.getNode()->getNumValues();
7307 SmallVector<SDValue, 4> Ops;
7309 Ops.push_back(ZExt);
7310 for (unsigned I = 1; I != NumVals; ++I)
7311 Ops.push_back(Op.getValue(I));
7313 return DAG.getMergeValues(Ops, SL);
7316 /// \brief Populate a CallLowerinInfo (into \p CLI) based on the properties of
7317 /// the call being lowered.
7319 /// This is a helper for lowering intrinsics that follow a target calling
7320 /// convention or require stack pointer adjustment. Only a subset of the
7321 /// intrinsic's operands need to participate in the calling convention.
7322 void SelectionDAGBuilder::populateCallLoweringInfo(
7323 TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
7324 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
7325 bool IsPatchPoint) {
7326 TargetLowering::ArgListTy Args;
7327 Args.reserve(NumArgs);
7329 // Populate the argument list.
7330 // Attributes for args start at offset 1, after the return attribute.
7331 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
7332 ArgI != ArgE; ++ArgI) {
7333 const Value *V = CS->getOperand(ArgI);
7335 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
7337 TargetLowering::ArgListEntry Entry;
7338 Entry.Node = getValue(V);
7339 Entry.Ty = V->getType();
7340 Entry.setAttributes(&CS, ArgIdx);
7341 Args.push_back(Entry);
7344 CLI.setDebugLoc(getCurSDLoc())
7345 .setChain(getRoot())
7346 .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
7347 .setDiscardResult(CS->use_empty())
7348 .setIsPatchPoint(IsPatchPoint);
7351 /// \brief Add a stack map intrinsic call's live variable operands to a stackmap
7352 /// or patchpoint target node's operand list.
7354 /// Constants are converted to TargetConstants purely as an optimization to
7355 /// avoid constant materialization and register allocation.
7357 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
7358 /// generate addess computation nodes, and so ExpandISelPseudo can convert the
7359 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
7360 /// address materialization and register allocation, but may also be required
7361 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
7362 /// alloca in the entry block, then the runtime may assume that the alloca's
7363 /// StackMap location can be read immediately after compilation and that the
7364 /// location is valid at any point during execution (this is similar to the
7365 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
7366 /// only available in a register, then the runtime would need to trap when
7367 /// execution reaches the StackMap in order to read the alloca's location.
7368 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
7369 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
7370 SelectionDAGBuilder &Builder) {
7371 for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
7372 SDValue OpVal = Builder.getValue(CS.getArgument(i));
7373 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
7375 Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
7377 Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
7378 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
7379 const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
7380 Ops.push_back(Builder.DAG.getTargetFrameIndex(
7381 FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
7383 Ops.push_back(OpVal);
7387 /// \brief Lower llvm.experimental.stackmap directly to its target opcode.
7388 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
7389 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
7390 // [live variables...])
7392 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
7394 SDValue Chain, InFlag, Callee, NullPtr;
7395 SmallVector<SDValue, 32> Ops;
7397 SDLoc DL = getCurSDLoc();
7398 Callee = getValue(CI.getCalledValue());
7399 NullPtr = DAG.getIntPtrConstant(0, DL, true);
7401 // The stackmap intrinsic only records the live variables (the arguemnts
7402 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
7403 // intrinsic, this won't be lowered to a function call. This means we don't
7404 // have to worry about calling conventions and target specific lowering code.
7405 // Instead we perform the call lowering right here.
7407 // chain, flag = CALLSEQ_START(chain, 0)
7408 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
7409 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
7411 Chain = DAG.getCALLSEQ_START(getRoot(), NullPtr, DL);
7412 InFlag = Chain.getValue(1);
7414 // Add the <id> and <numBytes> constants.
7415 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
7416 Ops.push_back(DAG.getTargetConstant(
7417 cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
7418 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
7419 Ops.push_back(DAG.getTargetConstant(
7420 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
7423 // Push live variables for the stack map.
7424 addStackMapLiveVars(&CI, 2, DL, Ops, *this);
7426 // We are not pushing any register mask info here on the operands list,
7427 // because the stackmap doesn't clobber anything.
7429 // Push the chain and the glue flag.
7430 Ops.push_back(Chain);
7431 Ops.push_back(InFlag);
7433 // Create the STACKMAP node.
7434 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7435 SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
7436 Chain = SDValue(SM, 0);
7437 InFlag = Chain.getValue(1);
7439 Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
7441 // Stackmaps don't generate values, so nothing goes into the NodeMap.
7443 // Set the root to the target-lowered call chain.
7446 // Inform the Frame Information that we have a stackmap in this function.
7447 FuncInfo.MF->getFrameInfo().setHasStackMap();
7450 /// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
7451 void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
7452 const BasicBlock *EHPadBB) {
7453 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
7458 // [live variables...])
7460 CallingConv::ID CC = CS.getCallingConv();
7461 bool IsAnyRegCC = CC == CallingConv::AnyReg;
7462 bool HasDef = !CS->getType()->isVoidTy();
7463 SDLoc dl = getCurSDLoc();
7464 SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
7466 // Handle immediate and symbolic callees.
7467 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
7468 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
7470 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
7471 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
7472 SDLoc(SymbolicCallee),
7473 SymbolicCallee->getValueType(0));
7475 // Get the real number of arguments participating in the call <numArgs>
7476 SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
7477 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
7479 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
7480 // Intrinsics include all meta-operands up to but not including CC.
7481 unsigned NumMetaOpers = PatchPointOpers::CCPos;
7482 assert(CS.arg_size() >= NumMetaOpers + NumArgs &&
7483 "Not enough arguments provided to the patchpoint intrinsic");
7485 // For AnyRegCC the arguments are lowered later on manually.
7486 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
7488 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
7490 TargetLowering::CallLoweringInfo CLI(DAG);
7491 populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
7493 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7495 SDNode *CallEnd = Result.second.getNode();
7496 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
7497 CallEnd = CallEnd->getOperand(0).getNode();
7499 /// Get a call instruction from the call sequence chain.
7500 /// Tail calls are not allowed.
7501 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
7502 "Expected a callseq node.");
7503 SDNode *Call = CallEnd->getOperand(0).getNode();
7504 bool HasGlue = Call->getGluedNode();
7506 // Replace the target specific call node with the patchable intrinsic.
7507 SmallVector<SDValue, 8> Ops;
7509 // Add the <id> and <numBytes> constants.
7510 SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
7511 Ops.push_back(DAG.getTargetConstant(
7512 cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
7513 SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
7514 Ops.push_back(DAG.getTargetConstant(
7515 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
7519 Ops.push_back(Callee);
7521 // Adjust <numArgs> to account for any arguments that have been passed on the
7523 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
7524 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
7525 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
7526 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
7528 // Add the calling convention
7529 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
7531 // Add the arguments we omitted previously. The register allocator should
7532 // place these in any free register.
7534 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
7535 Ops.push_back(getValue(CS.getArgument(i)));
7537 // Push the arguments from the call instruction up to the register mask.
7538 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
7539 Ops.append(Call->op_begin() + 2, e);
7541 // Push live variables for the stack map.
7542 addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
7544 // Push the register mask info.
7546 Ops.push_back(*(Call->op_end()-2));
7548 Ops.push_back(*(Call->op_end()-1));
7550 // Push the chain (this is originally the first operand of the call, but
7551 // becomes now the last or second to last operand).
7552 Ops.push_back(*(Call->op_begin()));
7554 // Push the glue flag (last operand).
7556 Ops.push_back(*(Call->op_end()-1));
7559 if (IsAnyRegCC && HasDef) {
7560 // Create the return types based on the intrinsic definition
7561 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7562 SmallVector<EVT, 3> ValueVTs;
7563 ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7564 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
7566 // There is always a chain and a glue type at the end
7567 ValueVTs.push_back(MVT::Other);
7568 ValueVTs.push_back(MVT::Glue);
7569 NodeTys = DAG.getVTList(ValueVTs);
7571 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7573 // Replace the target specific call node with a PATCHPOINT node.
7574 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
7577 // Update the NodeMap.
7580 setValue(CS.getInstruction(), SDValue(MN, 0));
7582 setValue(CS.getInstruction(), Result.first);
7585 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
7586 // call sequence. Furthermore the location of the chain and glue can change
7587 // when the AnyReg calling convention is used and the intrinsic returns a
7589 if (IsAnyRegCC && HasDef) {
7590 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
7591 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
7592 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
7594 DAG.ReplaceAllUsesWith(Call, MN);
7595 DAG.DeleteNode(Call);
7597 // Inform the Frame Information that we have a patchpoint in this function.
7598 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
7601 /// Returns an AttributeList representing the attributes applied to the return
7602 /// value of the given call.
7603 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
7604 SmallVector<Attribute::AttrKind, 2> Attrs;
7606 Attrs.push_back(Attribute::SExt);
7608 Attrs.push_back(Attribute::ZExt);
7610 Attrs.push_back(Attribute::InReg);
7612 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
7616 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
7617 /// implementation, which just calls LowerCall.
7618 /// FIXME: When all targets are
7619 /// migrated to using LowerCall, this hook should be integrated into SDISel.
7620 std::pair<SDValue, SDValue>
7621 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
7622 // Handle the incoming return values from the call.
7624 Type *OrigRetTy = CLI.RetTy;
7625 SmallVector<EVT, 4> RetTys;
7626 SmallVector<uint64_t, 4> Offsets;
7627 auto &DL = CLI.DAG.getDataLayout();
7628 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
7630 SmallVector<ISD::OutputArg, 4> Outs;
7631 GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
7633 bool CanLowerReturn =
7634 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
7635 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
7637 SDValue DemoteStackSlot;
7638 int DemoteStackIdx = -100;
7639 if (!CanLowerReturn) {
7640 // FIXME: equivalent assert?
7641 // assert(!CS.hasInAllocaArgument() &&
7642 // "sret demotion is incompatible with inalloca");
7643 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
7644 unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
7645 MachineFunction &MF = CLI.DAG.getMachineFunction();
7646 DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7647 Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
7649 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
7651 Entry.Node = DemoteStackSlot;
7652 Entry.Ty = StackSlotPtrType;
7653 Entry.IsSExt = false;
7654 Entry.IsZExt = false;
7655 Entry.IsInReg = false;
7656 Entry.IsSRet = true;
7657 Entry.IsNest = false;
7658 Entry.IsByVal = false;
7659 Entry.IsReturned = false;
7660 Entry.IsSwiftSelf = false;
7661 Entry.IsSwiftError = false;
7662 Entry.Alignment = Align;
7663 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
7664 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
7666 // sret demotion isn't compatible with tail-calls, since the sret argument
7667 // points into the callers stack frame.
7668 CLI.IsTailCall = false;
7670 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
7672 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
7673 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
7674 for (unsigned i = 0; i != NumRegs; ++i) {
7675 ISD::InputArg MyFlags;
7676 MyFlags.VT = RegisterVT;
7678 MyFlags.Used = CLI.IsReturnValueUsed;
7680 MyFlags.Flags.setSExt();
7682 MyFlags.Flags.setZExt();
7684 MyFlags.Flags.setInReg();
7685 CLI.Ins.push_back(MyFlags);
7690 // We push in swifterror return as the last element of CLI.Ins.
7691 ArgListTy &Args = CLI.getArgs();
7692 if (supportSwiftError()) {
7693 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
7694 if (Args[i].IsSwiftError) {
7695 ISD::InputArg MyFlags;
7696 MyFlags.VT = getPointerTy(DL);
7697 MyFlags.ArgVT = EVT(getPointerTy(DL));
7698 MyFlags.Flags.setSwiftError();
7699 CLI.Ins.push_back(MyFlags);
7704 // Handle all of the outgoing arguments.
7706 CLI.OutVals.clear();
7707 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
7708 SmallVector<EVT, 4> ValueVTs;
7709 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
7710 Type *FinalType = Args[i].Ty;
7711 if (Args[i].IsByVal)
7712 FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
7713 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
7714 FinalType, CLI.CallConv, CLI.IsVarArg);
7715 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
7717 EVT VT = ValueVTs[Value];
7718 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
7719 SDValue Op = SDValue(Args[i].Node.getNode(),
7720 Args[i].Node.getResNo() + Value);
7721 ISD::ArgFlagsTy Flags;
7722 unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
7728 if (Args[i].IsInReg) {
7729 // If we are using vectorcall calling convention, a structure that is
7730 // passed InReg - is surely an HVA
7731 if (CLI.CallConv == CallingConv::X86_VectorCall &&
7732 isa<StructType>(FinalType)) {
7733 // The first value of a structure is marked
7735 Flags.setHvaStart();
7743 if (Args[i].IsSwiftSelf)
7744 Flags.setSwiftSelf();
7745 if (Args[i].IsSwiftError)
7746 Flags.setSwiftError();
7747 if (Args[i].IsByVal)
7749 if (Args[i].IsInAlloca) {
7750 Flags.setInAlloca();
7751 // Set the byval flag for CCAssignFn callbacks that don't know about
7752 // inalloca. This way we can know how many bytes we should've allocated
7753 // and how many bytes a callee cleanup function will pop. If we port
7754 // inalloca to more targets, we'll have to add custom inalloca handling
7755 // in the various CC lowering callbacks.
7758 if (Args[i].IsByVal || Args[i].IsInAlloca) {
7759 PointerType *Ty = cast<PointerType>(Args[i].Ty);
7760 Type *ElementTy = Ty->getElementType();
7761 Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
7762 // For ByVal, alignment should come from FE. BE will guess if this
7763 // info is not there but there are cases it cannot get right.
7764 unsigned FrameAlign;
7765 if (Args[i].Alignment)
7766 FrameAlign = Args[i].Alignment;
7768 FrameAlign = getByValTypeAlignment(ElementTy, DL);
7769 Flags.setByValAlign(FrameAlign);
7774 Flags.setInConsecutiveRegs();
7775 Flags.setOrigAlign(OriginalAlignment);
7777 MVT PartVT = getRegisterType(CLI.RetTy->getContext(), VT);
7778 unsigned NumParts = getNumRegisters(CLI.RetTy->getContext(), VT);
7779 SmallVector<SDValue, 4> Parts(NumParts);
7780 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
7783 ExtendKind = ISD::SIGN_EXTEND;
7784 else if (Args[i].IsZExt)
7785 ExtendKind = ISD::ZERO_EXTEND;
7787 // Conservatively only handle 'returned' on non-vectors for now
7788 if (Args[i].IsReturned && !Op.getValueType().isVector()) {
7789 assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
7790 "unexpected use of 'returned'");
7791 // Before passing 'returned' to the target lowering code, ensure that
7792 // either the register MVT and the actual EVT are the same size or that
7793 // the return value and argument are extended in the same way; in these
7794 // cases it's safe to pass the argument register value unchanged as the
7795 // return register value (although it's at the target's option whether
7797 // TODO: allow code generation to take advantage of partially preserved
7798 // registers rather than clobbering the entire register when the
7799 // parameter extension method is not compatible with the return
7801 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
7802 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
7803 CLI.RetZExt == Args[i].IsZExt))
7804 Flags.setReturned();
7807 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
7808 CLI.CS ? CLI.CS->getInstruction() : nullptr, ExtendKind);
7810 for (unsigned j = 0; j != NumParts; ++j) {
7811 // if it isn't first piece, alignment must be 1
7812 ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
7813 i < CLI.NumFixedArgs,
7814 i, j*Parts[j].getValueType().getStoreSize());
7815 if (NumParts > 1 && j == 0)
7816 MyFlags.Flags.setSplit();
7818 MyFlags.Flags.setOrigAlign(1);
7819 if (j == NumParts - 1)
7820 MyFlags.Flags.setSplitEnd();
7823 CLI.Outs.push_back(MyFlags);
7824 CLI.OutVals.push_back(Parts[j]);
7827 if (NeedsRegBlock && Value == NumValues - 1)
7828 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
7832 SmallVector<SDValue, 4> InVals;
7833 CLI.Chain = LowerCall(CLI, InVals);
7835 // Update CLI.InVals to use outside of this function.
7836 CLI.InVals = InVals;
7838 // Verify that the target's LowerCall behaved as expected.
7839 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
7840 "LowerCall didn't return a valid chain!");
7841 assert((!CLI.IsTailCall || InVals.empty()) &&
7842 "LowerCall emitted a return value for a tail call!");
7843 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
7844 "LowerCall didn't emit the correct number of values!");
7846 // For a tail call, the return value is merely live-out and there aren't
7847 // any nodes in the DAG representing it. Return a special value to
7848 // indicate that a tail call has been emitted and no more Instructions
7849 // should be processed in the current block.
7850 if (CLI.IsTailCall) {
7851 CLI.DAG.setRoot(CLI.Chain);
7852 return std::make_pair(SDValue(), SDValue());
7856 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
7857 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
7858 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
7859 "LowerCall emitted a value with the wrong type!");
7863 SmallVector<SDValue, 4> ReturnValues;
7864 if (!CanLowerReturn) {
7865 // The instruction result is the result of loading from the
7866 // hidden sret parameter.
7867 SmallVector<EVT, 1> PVTs;
7868 Type *PtrRetTy = PointerType::getUnqual(OrigRetTy);
7870 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
7871 assert(PVTs.size() == 1 && "Pointers should fit in one register");
7872 EVT PtrVT = PVTs[0];
7874 unsigned NumValues = RetTys.size();
7875 ReturnValues.resize(NumValues);
7876 SmallVector<SDValue, 4> Chains(NumValues);
7878 // An aggregate return value cannot wrap around the address space, so
7879 // offsets to its parts don't wrap either.
7881 Flags.setNoUnsignedWrap(true);
7883 for (unsigned i = 0; i < NumValues; ++i) {
7884 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
7885 CLI.DAG.getConstant(Offsets[i], CLI.DL,
7887 SDValue L = CLI.DAG.getLoad(
7888 RetTys[i], CLI.DL, CLI.Chain, Add,
7889 MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
7890 DemoteStackIdx, Offsets[i]),
7891 /* Alignment = */ 1);
7892 ReturnValues[i] = L;
7893 Chains[i] = L.getValue(1);
7896 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
7898 // Collect the legal value parts into potentially illegal values
7899 // that correspond to the original function's return values.
7900 Optional<ISD::NodeType> AssertOp;
7902 AssertOp = ISD::AssertSext;
7903 else if (CLI.RetZExt)
7904 AssertOp = ISD::AssertZext;
7905 unsigned CurReg = 0;
7906 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
7908 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), VT);
7909 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
7911 ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
7912 NumRegs, RegisterVT, VT, nullptr,
7917 // For a function returning void, there is no return value. We can't create
7918 // such a node, so we just return a null return value in that case. In
7919 // that case, nothing will actually look at the value.
7920 if (ReturnValues.empty())
7921 return std::make_pair(SDValue(), CLI.Chain);
7924 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
7925 CLI.DAG.getVTList(RetTys), ReturnValues);
7926 return std::make_pair(Res, CLI.Chain);
7929 void TargetLowering::LowerOperationWrapper(SDNode *N,
7930 SmallVectorImpl<SDValue> &Results,
7931 SelectionDAG &DAG) const {
7932 if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
7933 Results.push_back(Res);
7936 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7937 llvm_unreachable("LowerOperation not implemented for this target!");
7941 SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
7942 SDValue Op = getNonRegisterValue(V);
7943 assert((Op.getOpcode() != ISD::CopyFromReg ||
7944 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
7945 "Copy from a reg to the same reg!");
7946 assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
7948 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7949 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
7951 SDValue Chain = DAG.getEntryNode();
7953 ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
7954 FuncInfo.PreferredExtendType.end())
7956 : FuncInfo.PreferredExtendType[V];
7957 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
7958 PendingExports.push_back(Chain);
7961 #include "llvm/CodeGen/SelectionDAGISel.h"
7963 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
7964 /// entry block, return true. This includes arguments used by switches, since
7965 /// the switch may expand into multiple basic blocks.
7966 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
7967 // With FastISel active, we may be splitting blocks, so force creation
7968 // of virtual registers for all non-dead arguments.
7970 return A->use_empty();
7972 const BasicBlock &Entry = A->getParent()->front();
7973 for (const User *U : A->users())
7974 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
7975 return false; // Use not in entry block.
7980 typedef DenseMap<const Argument *,
7981 std::pair<const AllocaInst *, const StoreInst *>>
7982 ArgCopyElisionMapTy;
7984 /// Scan the entry block of the function in FuncInfo for arguments that look
7985 /// like copies into a local alloca. Record any copied arguments in
7986 /// ArgCopyElisionCandidates.
7988 findArgumentCopyElisionCandidates(const DataLayout &DL,
7989 FunctionLoweringInfo *FuncInfo,
7990 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
7991 // Record the state of every static alloca used in the entry block. Argument
7992 // allocas are all used in the entry block, so we need approximately as many
7993 // entries as we have arguments.
7994 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
7995 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
7996 unsigned NumArgs = FuncInfo->Fn->arg_size();
7997 StaticAllocas.reserve(NumArgs * 2);
7999 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
8002 V = V->stripPointerCasts();
8003 const auto *AI = dyn_cast<AllocaInst>(V);
8004 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
8006 auto Iter = StaticAllocas.insert({AI, Unknown});
8007 return &Iter.first->second;
8010 // Look for stores of arguments to static allocas. Look through bitcasts and
8011 // GEPs to handle type coercions, as long as the alloca is fully initialized
8012 // by the store. Any non-store use of an alloca escapes it and any subsequent
8013 // unanalyzed store might write it.
8014 // FIXME: Handle structs initialized with multiple stores.
8015 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
8016 // Look for stores, and handle non-store uses conservatively.
8017 const auto *SI = dyn_cast<StoreInst>(&I);
8019 // We will look through cast uses, so ignore them completely.
8022 // Ignore debug info intrinsics, they don't escape or store to allocas.
8023 if (isa<DbgInfoIntrinsic>(I))
8025 // This is an unknown instruction. Assume it escapes or writes to all
8026 // static alloca operands.
8027 for (const Use &U : I.operands()) {
8028 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
8029 *Info = StaticAllocaInfo::Clobbered;
8034 // If the stored value is a static alloca, mark it as escaped.
8035 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
8036 *Info = StaticAllocaInfo::Clobbered;
8038 // Check if the destination is a static alloca.
8039 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
8040 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
8043 const AllocaInst *AI = cast<AllocaInst>(Dst);
8045 // Skip allocas that have been initialized or clobbered.
8046 if (*Info != StaticAllocaInfo::Unknown)
8049 // Check if the stored value is an argument, and that this store fully
8050 // initializes the alloca. Don't elide copies from the same argument twice.
8051 const Value *Val = SI->getValueOperand()->stripPointerCasts();
8052 const auto *Arg = dyn_cast<Argument>(Val);
8053 if (!Arg || Arg->hasInAllocaAttr() || Arg->hasByValAttr() ||
8054 Arg->getType()->isEmptyTy() ||
8055 DL.getTypeStoreSize(Arg->getType()) !=
8056 DL.getTypeAllocSize(AI->getAllocatedType()) ||
8057 ArgCopyElisionCandidates.count(Arg)) {
8058 *Info = StaticAllocaInfo::Clobbered;
8062 DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI << '\n');
8064 // Mark this alloca and store for argument copy elision.
8065 *Info = StaticAllocaInfo::Elidable;
8066 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
8068 // Stop scanning if we've seen all arguments. This will happen early in -O0
8069 // builds, which is useful, because -O0 builds have large entry blocks and
8071 if (ArgCopyElisionCandidates.size() == NumArgs)
8076 /// Try to elide argument copies from memory into a local alloca. Succeeds if
8077 /// ArgVal is a load from a suitable fixed stack object.
8078 static void tryToElideArgumentCopy(
8079 FunctionLoweringInfo *FuncInfo, SmallVectorImpl<SDValue> &Chains,
8080 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
8081 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
8082 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
8083 SDValue ArgVal, bool &ArgHasUses) {
8084 // Check if this is a load from a fixed stack object.
8085 auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
8088 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
8092 // Check that the fixed stack object is the right size and alignment.
8093 // Look at the alignment that the user wrote on the alloca instead of looking
8094 // at the stack object.
8095 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
8096 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
8097 const AllocaInst *AI = ArgCopyIter->second.first;
8098 int FixedIndex = FINode->getIndex();
8099 int &AllocaIndex = FuncInfo->StaticAllocaMap[AI];
8100 int OldIndex = AllocaIndex;
8101 MachineFrameInfo &MFI = FuncInfo->MF->getFrameInfo();
8102 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
8103 DEBUG(dbgs() << " argument copy elision failed due to bad fixed stack "
8107 unsigned RequiredAlignment = AI->getAlignment();
8108 if (!RequiredAlignment) {
8109 RequiredAlignment = FuncInfo->MF->getDataLayout().getABITypeAlignment(
8110 AI->getAllocatedType());
8112 if (MFI.getObjectAlignment(FixedIndex) < RequiredAlignment) {
8113 DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
8114 "greater than stack argument alignment ("
8115 << RequiredAlignment << " vs "
8116 << MFI.getObjectAlignment(FixedIndex) << ")\n");
8120 // Perform the elision. Delete the old stack object and replace its only use
8121 // in the variable info map. Mark the stack object as mutable.
8123 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
8124 << " Replacing frame index " << OldIndex << " with " << FixedIndex
8127 MFI.RemoveStackObject(OldIndex);
8128 MFI.setIsImmutableObjectIndex(FixedIndex, false);
8129 AllocaIndex = FixedIndex;
8130 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
8131 Chains.push_back(ArgVal.getValue(1));
8133 // Avoid emitting code for the store implementing the copy.
8134 const StoreInst *SI = ArgCopyIter->second.second;
8135 ElidedArgCopyInstrs.insert(SI);
8137 // Check for uses of the argument again so that we can avoid exporting ArgVal
8138 // if it is't used by anything other than the store.
8139 for (const Value *U : Arg.users()) {
8147 void SelectionDAGISel::LowerArguments(const Function &F) {
8148 SelectionDAG &DAG = SDB->DAG;
8149 SDLoc dl = SDB->getCurSDLoc();
8150 const DataLayout &DL = DAG.getDataLayout();
8151 SmallVector<ISD::InputArg, 16> Ins;
8153 if (!FuncInfo->CanLowerReturn) {
8154 // Put in an sret pointer parameter before all the other parameters.
8155 SmallVector<EVT, 1> ValueVTs;
8156 ComputeValueVTs(*TLI, DAG.getDataLayout(),
8157 PointerType::getUnqual(F.getReturnType()), ValueVTs);
8159 // NOTE: Assuming that a pointer will never break down to more than one VT
8161 ISD::ArgFlagsTy Flags;
8163 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
8164 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
8165 ISD::InputArg::NoArgIndex, 0);
8166 Ins.push_back(RetArg);
8169 // Look for stores of arguments to static allocas. Mark such arguments with a
8170 // flag to ask the target to give us the memory location of that argument if
8172 ArgCopyElisionMapTy ArgCopyElisionCandidates;
8173 findArgumentCopyElisionCandidates(DL, FuncInfo, ArgCopyElisionCandidates);
8175 // Set up the incoming argument description vector.
8176 for (const Argument &Arg : F.args()) {
8177 unsigned ArgNo = Arg.getArgNo();
8178 SmallVector<EVT, 4> ValueVTs;
8179 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
8180 bool isArgValueUsed = !Arg.use_empty();
8181 unsigned PartBase = 0;
8182 Type *FinalType = Arg.getType();
8183 if (Arg.hasAttribute(Attribute::ByVal))
8184 FinalType = cast<PointerType>(FinalType)->getElementType();
8185 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
8186 FinalType, F.getCallingConv(), F.isVarArg());
8187 for (unsigned Value = 0, NumValues = ValueVTs.size();
8188 Value != NumValues; ++Value) {
8189 EVT VT = ValueVTs[Value];
8190 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
8191 ISD::ArgFlagsTy Flags;
8192 unsigned OriginalAlignment = DL.getABITypeAlignment(ArgTy);
8194 if (Arg.hasAttribute(Attribute::ZExt))
8196 if (Arg.hasAttribute(Attribute::SExt))
8198 if (Arg.hasAttribute(Attribute::InReg)) {
8199 // If we are using vectorcall calling convention, a structure that is
8200 // passed InReg - is surely an HVA
8201 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
8202 isa<StructType>(Arg.getType())) {
8203 // The first value of a structure is marked
8205 Flags.setHvaStart();
8211 if (Arg.hasAttribute(Attribute::StructRet))
8213 if (Arg.hasAttribute(Attribute::SwiftSelf))
8214 Flags.setSwiftSelf();
8215 if (Arg.hasAttribute(Attribute::SwiftError))
8216 Flags.setSwiftError();
8217 if (Arg.hasAttribute(Attribute::ByVal))
8219 if (Arg.hasAttribute(Attribute::InAlloca)) {
8220 Flags.setInAlloca();
8221 // Set the byval flag for CCAssignFn callbacks that don't know about
8222 // inalloca. This way we can know how many bytes we should've allocated
8223 // and how many bytes a callee cleanup function will pop. If we port
8224 // inalloca to more targets, we'll have to add custom inalloca handling
8225 // in the various CC lowering callbacks.
8228 if (F.getCallingConv() == CallingConv::X86_INTR) {
8229 // IA Interrupt passes frame (1st parameter) by value in the stack.
8233 if (Flags.isByVal() || Flags.isInAlloca()) {
8234 PointerType *Ty = cast<PointerType>(Arg.getType());
8235 Type *ElementTy = Ty->getElementType();
8236 Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8237 // For ByVal, alignment should be passed from FE. BE will guess if
8238 // this info is not there but there are cases it cannot get right.
8239 unsigned FrameAlign;
8240 if (Arg.getParamAlignment())
8241 FrameAlign = Arg.getParamAlignment();
8243 FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
8244 Flags.setByValAlign(FrameAlign);
8246 if (Arg.hasAttribute(Attribute::Nest))
8249 Flags.setInConsecutiveRegs();
8250 Flags.setOrigAlign(OriginalAlignment);
8251 if (ArgCopyElisionCandidates.count(&Arg))
8252 Flags.setCopyElisionCandidate();
8254 MVT RegisterVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8255 unsigned NumRegs = TLI->getNumRegisters(*CurDAG->getContext(), VT);
8256 for (unsigned i = 0; i != NumRegs; ++i) {
8257 ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
8258 ArgNo, PartBase+i*RegisterVT.getStoreSize());
8259 if (NumRegs > 1 && i == 0)
8260 MyFlags.Flags.setSplit();
8261 // if it isn't first piece, alignment must be 1
8263 MyFlags.Flags.setOrigAlign(1);
8264 if (i == NumRegs - 1)
8265 MyFlags.Flags.setSplitEnd();
8267 Ins.push_back(MyFlags);
8269 if (NeedsRegBlock && Value == NumValues - 1)
8270 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
8271 PartBase += VT.getStoreSize();
8275 // Call the target to set up the argument values.
8276 SmallVector<SDValue, 8> InVals;
8277 SDValue NewRoot = TLI->LowerFormalArguments(
8278 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
8280 // Verify that the target's LowerFormalArguments behaved as expected.
8281 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
8282 "LowerFormalArguments didn't return a valid chain!");
8283 assert(InVals.size() == Ins.size() &&
8284 "LowerFormalArguments didn't emit the correct number of values!");
8286 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
8287 assert(InVals[i].getNode() &&
8288 "LowerFormalArguments emitted a null value!");
8289 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
8290 "LowerFormalArguments emitted a value with the wrong type!");
8294 // Update the DAG with the new chain value resulting from argument lowering.
8295 DAG.setRoot(NewRoot);
8297 // Set up the argument values.
8299 if (!FuncInfo->CanLowerReturn) {
8300 // Create a virtual register for the sret pointer, and put in a copy
8301 // from the sret argument into it.
8302 SmallVector<EVT, 1> ValueVTs;
8303 ComputeValueVTs(*TLI, DAG.getDataLayout(),
8304 PointerType::getUnqual(F.getReturnType()), ValueVTs);
8305 MVT VT = ValueVTs[0].getSimpleVT();
8306 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8307 Optional<ISD::NodeType> AssertOp = None;
8308 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1,
8309 RegVT, VT, nullptr, AssertOp);
8311 MachineFunction& MF = SDB->DAG.getMachineFunction();
8312 MachineRegisterInfo& RegInfo = MF.getRegInfo();
8313 unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
8314 FuncInfo->DemoteRegister = SRetReg;
8316 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
8317 DAG.setRoot(NewRoot);
8319 // i indexes lowered arguments. Bump it past the hidden sret argument.
8323 SmallVector<SDValue, 4> Chains;
8324 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
8325 for (const Argument &Arg : F.args()) {
8326 SmallVector<SDValue, 4> ArgValues;
8327 SmallVector<EVT, 4> ValueVTs;
8328 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
8329 unsigned NumValues = ValueVTs.size();
8333 bool ArgHasUses = !Arg.use_empty();
8335 // Elide the copying store if the target loaded this argument from a
8336 // suitable fixed stack object.
8337 if (Ins[i].Flags.isCopyElisionCandidate()) {
8338 tryToElideArgumentCopy(FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
8339 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
8340 InVals[i], ArgHasUses);
8343 // If this argument is unused then remember its value. It is used to generate
8344 // debugging information.
8345 bool isSwiftErrorArg =
8346 TLI->supportSwiftError() &&
8347 Arg.hasAttribute(Attribute::SwiftError);
8348 if (!ArgHasUses && !isSwiftErrorArg) {
8349 SDB->setUnusedArgValue(&Arg, InVals[i]);
8351 // Also remember any frame index for use in FastISel.
8352 if (FrameIndexSDNode *FI =
8353 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
8354 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
8357 for (unsigned Val = 0; Val != NumValues; ++Val) {
8358 EVT VT = ValueVTs[Val];
8359 MVT PartVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8360 unsigned NumParts = TLI->getNumRegisters(*CurDAG->getContext(), VT);
8362 // Even an apparant 'unused' swifterror argument needs to be returned. So
8363 // we do generate a copy for it that can be used on return from the
8365 if (ArgHasUses || isSwiftErrorArg) {
8366 Optional<ISD::NodeType> AssertOp;
8367 if (Arg.hasAttribute(Attribute::SExt))
8368 AssertOp = ISD::AssertSext;
8369 else if (Arg.hasAttribute(Attribute::ZExt))
8370 AssertOp = ISD::AssertZext;
8372 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
8373 PartVT, VT, nullptr, AssertOp));
8379 // We don't need to do anything else for unused arguments.
8380 if (ArgValues.empty())
8383 // Note down frame index.
8384 if (FrameIndexSDNode *FI =
8385 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
8386 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
8388 SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
8389 SDB->getCurSDLoc());
8391 SDB->setValue(&Arg, Res);
8392 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
8393 if (LoadSDNode *LNode =
8394 dyn_cast<LoadSDNode>(Res.getOperand(0).getNode()))
8395 if (FrameIndexSDNode *FI =
8396 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
8397 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
8400 // Update the SwiftErrorVRegDefMap.
8401 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
8402 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
8403 if (TargetRegisterInfo::isVirtualRegister(Reg))
8404 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB,
8405 FuncInfo->SwiftErrorArg, Reg);
8408 // If this argument is live outside of the entry block, insert a copy from
8409 // wherever we got it to the vreg that other BB's will reference it as.
8410 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
8411 // If we can, though, try to skip creating an unnecessary vreg.
8412 // FIXME: This isn't very clean... it would be nice to make this more
8413 // general. It's also subtly incompatible with the hacks FastISel
8415 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
8416 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
8417 FuncInfo->ValueMap[&Arg] = Reg;
8421 if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
8422 FuncInfo->InitializeRegForValue(&Arg);
8423 SDB->CopyToExportRegsIfNeeded(&Arg);
8427 if (!Chains.empty()) {
8428 Chains.push_back(NewRoot);
8429 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
8432 DAG.setRoot(NewRoot);
8434 assert(i == InVals.size() && "Argument register count mismatch!");
8436 // If any argument copy elisions occurred and we have debug info, update the
8437 // stale frame indices used in the dbg.declare variable info table.
8438 MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
8439 if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
8440 for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
8441 auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
8442 if (I != ArgCopyElisionFrameIndexMap.end())
8443 VI.Slot = I->second;
8447 // Finally, if the target has anything special to do, allow it to do so.
8448 EmitFunctionEntryCode();
8451 /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
8452 /// ensure constants are generated when needed. Remember the virtual registers
8453 /// that need to be added to the Machine PHI nodes as input. We cannot just
8454 /// directly add them, because expansion might result in multiple MBB's for one
8455 /// BB. As such, the start of the BB might correspond to a different MBB than
8459 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
8460 const TerminatorInst *TI = LLVMBB->getTerminator();
8462 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
8464 // Check PHI nodes in successors that expect a value to be available from this
8466 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
8467 const BasicBlock *SuccBB = TI->getSuccessor(succ);
8468 if (!isa<PHINode>(SuccBB->begin())) continue;
8469 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
8471 // If this terminator has multiple identical successors (common for
8472 // switches), only handle each succ once.
8473 if (!SuccsHandled.insert(SuccMBB).second)
8476 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
8478 // At this point we know that there is a 1-1 correspondence between LLVM PHI
8479 // nodes and Machine PHI nodes, but the incoming operands have not been
8481 for (BasicBlock::const_iterator I = SuccBB->begin();
8482 const PHINode *PN = dyn_cast<PHINode>(I); ++I) {
8483 // Ignore dead phi's.
8484 if (PN->use_empty()) continue;
8487 if (PN->getType()->isEmptyTy())
8491 const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
8493 if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
8494 unsigned &RegOut = ConstantsOut[C];
8496 RegOut = FuncInfo.CreateRegs(C->getType());
8497 CopyValueToVirtualRegister(C, RegOut);
8501 DenseMap<const Value *, unsigned>::iterator I =
8502 FuncInfo.ValueMap.find(PHIOp);
8503 if (I != FuncInfo.ValueMap.end())
8506 assert(isa<AllocaInst>(PHIOp) &&
8507 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
8508 "Didn't codegen value into a register!??");
8509 Reg = FuncInfo.CreateRegs(PHIOp->getType());
8510 CopyValueToVirtualRegister(PHIOp, Reg);
8514 // Remember that this register needs to added to the machine PHI node as
8515 // the input for this MBB.
8516 SmallVector<EVT, 4> ValueVTs;
8517 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8518 ComputeValueVTs(TLI, DAG.getDataLayout(), PN->getType(), ValueVTs);
8519 for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
8520 EVT VT = ValueVTs[vti];
8521 unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
8522 for (unsigned i = 0, e = NumRegisters; i != e; ++i)
8523 FuncInfo.PHINodesToUpdate.push_back(
8524 std::make_pair(&*MBBI++, Reg + i));
8525 Reg += NumRegisters;
8530 ConstantsOut.clear();
8533 /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
8536 SelectionDAGBuilder::StackProtectorDescriptor::
8537 AddSuccessorMBB(const BasicBlock *BB,
8538 MachineBasicBlock *ParentMBB,
8540 MachineBasicBlock *SuccMBB) {
8541 // If SuccBB has not been created yet, create it.
8543 MachineFunction *MF = ParentMBB->getParent();
8544 MachineFunction::iterator BBI(ParentMBB);
8545 SuccMBB = MF->CreateMachineBasicBlock(BB);
8546 MF->insert(++BBI, SuccMBB);
8548 // Add it as a successor of ParentMBB.
8549 ParentMBB->addSuccessor(
8550 SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely));
8554 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
8555 MachineFunction::iterator I(MBB);
8556 if (++I == FuncInfo.MF->end())
8561 /// During lowering new call nodes can be created (such as memset, etc.).
8562 /// Those will become new roots of the current DAG, but complications arise
8563 /// when they are tail calls. In such cases, the call lowering will update
8564 /// the root, but the builder still needs to know that a tail call has been
8565 /// lowered in order to avoid generating an additional return.
8566 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
8567 // If the node is null, we do have a tail call.
8568 if (MaybeTC.getNode() != nullptr)
8569 DAG.setRoot(MaybeTC);
8575 SelectionDAGBuilder::getJumpTableRange(const CaseClusterVector &Clusters,
8576 unsigned First, unsigned Last) const {
8577 assert(Last >= First);
8578 const APInt &LowCase = Clusters[First].Low->getValue();
8579 const APInt &HighCase = Clusters[Last].High->getValue();
8580 assert(LowCase.getBitWidth() == HighCase.getBitWidth());
8582 // FIXME: A range of consecutive cases has 100% density, but only requires one
8583 // comparison to lower. We should discriminate against such consecutive ranges
8586 return (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100) + 1;
8589 uint64_t SelectionDAGBuilder::getJumpTableNumCases(
8590 const SmallVectorImpl<unsigned> &TotalCases, unsigned First,
8591 unsigned Last) const {
8592 assert(Last >= First);
8593 assert(TotalCases[Last] >= TotalCases[First]);
8595 TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
8599 bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters,
8600 unsigned First, unsigned Last,
8601 const SwitchInst *SI,
8602 MachineBasicBlock *DefaultMBB,
8603 CaseCluster &JTCluster) {
8604 assert(First <= Last);
8606 auto Prob = BranchProbability::getZero();
8607 unsigned NumCmps = 0;
8608 std::vector<MachineBasicBlock*> Table;
8609 DenseMap<MachineBasicBlock*, BranchProbability> JTProbs;
8611 // Initialize probabilities in JTProbs.
8612 for (unsigned I = First; I <= Last; ++I)
8613 JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
8615 for (unsigned I = First; I <= Last; ++I) {
8616 assert(Clusters[I].Kind == CC_Range);
8617 Prob += Clusters[I].Prob;
8618 const APInt &Low = Clusters[I].Low->getValue();
8619 const APInt &High = Clusters[I].High->getValue();
8620 NumCmps += (Low == High) ? 1 : 2;
8622 // Fill the gap between this and the previous cluster.
8623 const APInt &PreviousHigh = Clusters[I - 1].High->getValue();
8624 assert(PreviousHigh.slt(Low));
8625 uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
8626 for (uint64_t J = 0; J < Gap; J++)
8627 Table.push_back(DefaultMBB);
8629 uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
8630 for (uint64_t J = 0; J < ClusterSize; ++J)
8631 Table.push_back(Clusters[I].MBB);
8632 JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
8635 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8636 unsigned NumDests = JTProbs.size();
8637 if (TLI.isSuitableForBitTests(
8638 NumDests, NumCmps, Clusters[First].Low->getValue(),
8639 Clusters[Last].High->getValue(), DAG.getDataLayout())) {
8640 // Clusters[First..Last] should be lowered as bit tests instead.
8644 // Create the MBB that will load from and jump through the table.
8645 // Note: We create it here, but it's not inserted into the function yet.
8646 MachineFunction *CurMF = FuncInfo.MF;
8647 MachineBasicBlock *JumpTableMBB =
8648 CurMF->CreateMachineBasicBlock(SI->getParent());
8650 // Add successors. Note: use table order for determinism.
8651 SmallPtrSet<MachineBasicBlock *, 8> Done;
8652 for (MachineBasicBlock *Succ : Table) {
8653 if (Done.count(Succ))
8655 addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
8658 JumpTableMBB->normalizeSuccProbs();
8660 unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
8661 ->createJumpTableIndex(Table);
8663 // Set up the jump table info.
8664 JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
8665 JumpTableHeader JTH(Clusters[First].Low->getValue(),
8666 Clusters[Last].High->getValue(), SI->getCondition(),
8668 JTCases.emplace_back(std::move(JTH), std::move(JT));
8670 JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
8671 JTCases.size() - 1, Prob);
8675 void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
8676 const SwitchInst *SI,
8677 MachineBasicBlock *DefaultMBB) {
8679 // Clusters must be non-empty, sorted, and only contain Range clusters.
8680 assert(!Clusters.empty());
8681 for (CaseCluster &C : Clusters)
8682 assert(C.Kind == CC_Range);
8683 for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
8684 assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
8687 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8688 if (!TLI.areJTsAllowed(SI->getParent()->getParent()))
8691 const int64_t N = Clusters.size();
8692 const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries();
8693 const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2;
8695 if (N < 2 || N < MinJumpTableEntries)
8698 // TotalCases[i]: Total nbr of cases in Clusters[0..i].
8699 SmallVector<unsigned, 8> TotalCases(N);
8700 for (unsigned i = 0; i < N; ++i) {
8701 const APInt &Hi = Clusters[i].High->getValue();
8702 const APInt &Lo = Clusters[i].Low->getValue();
8703 TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
8705 TotalCases[i] += TotalCases[i - 1];
8708 // Cheap case: the whole range may be suitable for jump table.
8709 uint64_t Range = getJumpTableRange(Clusters,0, N - 1);
8710 uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1);
8711 assert(NumCases < UINT64_MAX / 100);
8712 assert(Range >= NumCases);
8713 if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
8714 CaseCluster JTCluster;
8715 if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
8716 Clusters[0] = JTCluster;
8722 // The algorithm below is not suitable for -O0.
8723 if (TM.getOptLevel() == CodeGenOpt::None)
8726 // Split Clusters into minimum number of dense partitions. The algorithm uses
8727 // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
8728 // for the Case Statement'" (1994), but builds the MinPartitions array in
8729 // reverse order to make it easier to reconstruct the partitions in ascending
8730 // order. In the choice between two optimal partitionings, it picks the one
8731 // which yields more jump tables.
8733 // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
8734 SmallVector<unsigned, 8> MinPartitions(N);
8735 // LastElement[i] is the last element of the partition starting at i.
8736 SmallVector<unsigned, 8> LastElement(N);
8737 // PartitionsScore[i] is used to break ties when choosing between two
8738 // partitionings resulting in the same number of partitions.
8739 SmallVector<unsigned, 8> PartitionsScore(N);
8740 // For PartitionsScore, a small number of comparisons is considered as good as
8741 // a jump table and a single comparison is considered better than a jump
8743 enum PartitionScores : unsigned {
8750 // Base case: There is only one way to partition Clusters[N-1].
8751 MinPartitions[N - 1] = 1;
8752 LastElement[N - 1] = N - 1;
8753 PartitionsScore[N - 1] = PartitionScores::SingleCase;
8755 // Note: loop indexes are signed to avoid underflow.
8756 for (int64_t i = N - 2; i >= 0; i--) {
8757 // Find optimal partitioning of Clusters[i..N-1].
8758 // Baseline: Put Clusters[i] into a partition on its own.
8759 MinPartitions[i] = MinPartitions[i + 1] + 1;
8761 PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase;
8763 // Search for a solution that results in fewer partitions.
8764 for (int64_t j = N - 1; j > i; j--) {
8765 // Try building a partition from Clusters[i..j].
8766 uint64_t Range = getJumpTableRange(Clusters, i, j);
8767 uint64_t NumCases = getJumpTableNumCases(TotalCases, i, j);
8768 assert(NumCases < UINT64_MAX / 100);
8769 assert(Range >= NumCases);
8770 if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
8771 unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
8772 unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1];
8773 int64_t NumEntries = j - i + 1;
8775 if (NumEntries == 1)
8776 Score += PartitionScores::SingleCase;
8777 else if (NumEntries <= SmallNumberOfEntries)
8778 Score += PartitionScores::FewCases;
8779 else if (NumEntries >= MinJumpTableEntries)
8780 Score += PartitionScores::Table;
8782 // If this leads to fewer partitions, or to the same number of
8783 // partitions with better score, it is a better partitioning.
8784 if (NumPartitions < MinPartitions[i] ||
8785 (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) {
8786 MinPartitions[i] = NumPartitions;
8788 PartitionsScore[i] = Score;
8794 // Iterate over the partitions, replacing some with jump tables in-place.
8795 unsigned DstIndex = 0;
8796 for (unsigned First = 0, Last; First < N; First = Last + 1) {
8797 Last = LastElement[First];
8798 assert(Last >= First);
8799 assert(DstIndex <= First);
8800 unsigned NumClusters = Last - First + 1;
8802 CaseCluster JTCluster;
8803 if (NumClusters >= MinJumpTableEntries &&
8804 buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
8805 Clusters[DstIndex++] = JTCluster;
8807 for (unsigned I = First; I <= Last; ++I)
8808 std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
8811 Clusters.resize(DstIndex);
8814 bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
8815 unsigned First, unsigned Last,
8816 const SwitchInst *SI,
8817 CaseCluster &BTCluster) {
8818 assert(First <= Last);
8822 BitVector Dests(FuncInfo.MF->getNumBlockIDs());
8823 unsigned NumCmps = 0;
8824 for (int64_t I = First; I <= Last; ++I) {
8825 assert(Clusters[I].Kind == CC_Range);
8826 Dests.set(Clusters[I].MBB->getNumber());
8827 NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
8829 unsigned NumDests = Dests.count();
8831 APInt Low = Clusters[First].Low->getValue();
8832 APInt High = Clusters[Last].High->getValue();
8833 assert(Low.slt(High));
8835 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8836 const DataLayout &DL = DAG.getDataLayout();
8837 if (!TLI.isSuitableForBitTests(NumDests, NumCmps, Low, High, DL))
8843 const int BitWidth = TLI.getPointerTy(DL).getSizeInBits();
8844 assert(TLI.rangeFitsInWord(Low, High, DL) &&
8845 "Case range must fit in bit mask!");
8847 // Check if the clusters cover a contiguous range such that no value in the
8848 // range will jump to the default statement.
8849 bool ContiguousRange = true;
8850 for (int64_t I = First + 1; I <= Last; ++I) {
8851 if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) {
8852 ContiguousRange = false;
8857 if (Low.isStrictlyPositive() && High.slt(BitWidth)) {
8858 // Optimize the case where all the case values fit in a word without having
8859 // to subtract minValue. In this case, we can optimize away the subtraction.
8860 LowBound = APInt::getNullValue(Low.getBitWidth());
8862 ContiguousRange = false;
8865 CmpRange = High - Low;
8869 auto TotalProb = BranchProbability::getZero();
8870 for (unsigned i = First; i <= Last; ++i) {
8871 // Find the CaseBits for this destination.
8873 for (j = 0; j < CBV.size(); ++j)
8874 if (CBV[j].BB == Clusters[i].MBB)
8876 if (j == CBV.size())
8878 CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
8879 CaseBits *CB = &CBV[j];
8881 // Update Mask, Bits and ExtraProb.
8882 uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
8883 uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
8884 assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
8885 CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
8886 CB->Bits += Hi - Lo + 1;
8887 CB->ExtraProb += Clusters[i].Prob;
8888 TotalProb += Clusters[i].Prob;
8892 std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
8893 // Sort by probability first, number of bits second.
8894 if (a.ExtraProb != b.ExtraProb)
8895 return a.ExtraProb > b.ExtraProb;
8896 return a.Bits > b.Bits;
8899 for (auto &CB : CBV) {
8900 MachineBasicBlock *BitTestBB =
8901 FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
8902 BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
8904 BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
8905 SI->getCondition(), -1U, MVT::Other, false,
8906 ContiguousRange, nullptr, nullptr, std::move(BTI),
8909 BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
8910 BitTestCases.size() - 1, TotalProb);
8914 void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
8915 const SwitchInst *SI) {
8916 // Partition Clusters into as few subsets as possible, where each subset has a
8917 // range that fits in a machine word and has <= 3 unique destinations.
8920 // Clusters must be sorted and contain Range or JumpTable clusters.
8921 assert(!Clusters.empty());
8922 assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
8923 for (const CaseCluster &C : Clusters)
8924 assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
8925 for (unsigned i = 1; i < Clusters.size(); ++i)
8926 assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
8929 // The algorithm below is not suitable for -O0.
8930 if (TM.getOptLevel() == CodeGenOpt::None)
8933 // If target does not have legal shift left, do not emit bit tests at all.
8934 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8935 const DataLayout &DL = DAG.getDataLayout();
8937 EVT PTy = TLI.getPointerTy(DL);
8938 if (!TLI.isOperationLegal(ISD::SHL, PTy))
8941 int BitWidth = PTy.getSizeInBits();
8942 const int64_t N = Clusters.size();
8944 // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
8945 SmallVector<unsigned, 8> MinPartitions(N);
8946 // LastElement[i] is the last element of the partition starting at i.
8947 SmallVector<unsigned, 8> LastElement(N);
8949 // FIXME: This might not be the best algorithm for finding bit test clusters.
8951 // Base case: There is only one way to partition Clusters[N-1].
8952 MinPartitions[N - 1] = 1;
8953 LastElement[N - 1] = N - 1;
8955 // Note: loop indexes are signed to avoid underflow.
8956 for (int64_t i = N - 2; i >= 0; --i) {
8957 // Find optimal partitioning of Clusters[i..N-1].
8958 // Baseline: Put Clusters[i] into a partition on its own.
8959 MinPartitions[i] = MinPartitions[i + 1] + 1;
8962 // Search for a solution that results in fewer partitions.
8963 // Note: the search is limited by BitWidth, reducing time complexity.
8964 for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
8965 // Try building a partition from Clusters[i..j].
8968 if (!TLI.rangeFitsInWord(Clusters[i].Low->getValue(),
8969 Clusters[j].High->getValue(), DL))
8972 // Check nbr of destinations and cluster types.
8973 // FIXME: This works, but doesn't seem very efficient.
8974 bool RangesOnly = true;
8975 BitVector Dests(FuncInfo.MF->getNumBlockIDs());
8976 for (int64_t k = i; k <= j; k++) {
8977 if (Clusters[k].Kind != CC_Range) {
8981 Dests.set(Clusters[k].MBB->getNumber());
8983 if (!RangesOnly || Dests.count() > 3)
8986 // Check if it's a better partition.
8987 unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
8988 if (NumPartitions < MinPartitions[i]) {
8989 // Found a better partition.
8990 MinPartitions[i] = NumPartitions;
8996 // Iterate over the partitions, replacing with bit-test clusters in-place.
8997 unsigned DstIndex = 0;
8998 for (unsigned First = 0, Last; First < N; First = Last + 1) {
8999 Last = LastElement[First];
9000 assert(First <= Last);
9001 assert(DstIndex <= First);
9003 CaseCluster BitTestCluster;
9004 if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
9005 Clusters[DstIndex++] = BitTestCluster;
9007 size_t NumClusters = Last - First + 1;
9008 std::memmove(&Clusters[DstIndex], &Clusters[First],
9009 sizeof(Clusters[0]) * NumClusters);
9010 DstIndex += NumClusters;
9013 Clusters.resize(DstIndex);
9016 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
9017 MachineBasicBlock *SwitchMBB,
9018 MachineBasicBlock *DefaultMBB) {
9019 MachineFunction *CurMF = FuncInfo.MF;
9020 MachineBasicBlock *NextMBB = nullptr;
9021 MachineFunction::iterator BBI(W.MBB);
9022 if (++BBI != FuncInfo.MF->end())
9025 unsigned Size = W.LastCluster - W.FirstCluster + 1;
9027 BranchProbabilityInfo *BPI = FuncInfo.BPI;
9029 if (Size == 2 && W.MBB == SwitchMBB) {
9030 // If any two of the cases has the same destination, and if one value
9031 // is the same as the other, but has one bit unset that the other has set,
9032 // use bit manipulation to do two compares at once. For example:
9033 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
9034 // TODO: This could be extended to merge any 2 cases in switches with 3
9036 // TODO: Handle cases where W.CaseBB != SwitchBB.
9037 CaseCluster &Small = *W.FirstCluster;
9038 CaseCluster &Big = *W.LastCluster;
9040 if (Small.Low == Small.High && Big.Low == Big.High &&
9041 Small.MBB == Big.MBB) {
9042 const APInt &SmallValue = Small.Low->getValue();
9043 const APInt &BigValue = Big.Low->getValue();
9045 // Check that there is only one bit different.
9046 APInt CommonBit = BigValue ^ SmallValue;
9047 if (CommonBit.isPowerOf2()) {
9048 SDValue CondLHS = getValue(Cond);
9049 EVT VT = CondLHS.getValueType();
9050 SDLoc DL = getCurSDLoc();
9052 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
9053 DAG.getConstant(CommonBit, DL, VT));
9054 SDValue Cond = DAG.getSetCC(
9055 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
9058 // Update successor info.
9059 // Both Small and Big will jump to Small.BB, so we sum up the
9061 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
9063 addSuccessorWithProb(
9064 SwitchMBB, DefaultMBB,
9065 // The default destination is the first successor in IR.
9066 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
9068 addSuccessorWithProb(SwitchMBB, DefaultMBB);
9070 // Insert the true branch.
9072 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
9073 DAG.getBasicBlock(Small.MBB));
9074 // Insert the false branch.
9075 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
9076 DAG.getBasicBlock(DefaultMBB));
9078 DAG.setRoot(BrCond);
9084 if (TM.getOptLevel() != CodeGenOpt::None) {
9085 // Order cases by probability so the most likely case will be checked first.
9086 std::sort(W.FirstCluster, W.LastCluster + 1,
9087 [](const CaseCluster &a, const CaseCluster &b) {
9088 return a.Prob > b.Prob;
9091 // Rearrange the case blocks so that the last one falls through if possible
9092 // without without changing the order of probabilities.
9093 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
9095 if (I->Prob > W.LastCluster->Prob)
9097 if (I->Kind == CC_Range && I->MBB == NextMBB) {
9098 std::swap(*I, *W.LastCluster);
9104 // Compute total probability.
9105 BranchProbability DefaultProb = W.DefaultProb;
9106 BranchProbability UnhandledProbs = DefaultProb;
9107 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
9108 UnhandledProbs += I->Prob;
9110 MachineBasicBlock *CurMBB = W.MBB;
9111 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
9112 MachineBasicBlock *Fallthrough;
9113 if (I == W.LastCluster) {
9114 // For the last cluster, fall through to the default destination.
9115 Fallthrough = DefaultMBB;
9117 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
9118 CurMF->insert(BBI, Fallthrough);
9119 // Put Cond in a virtual register to make it available from the new blocks.
9120 ExportFromCurrentBlock(Cond);
9122 UnhandledProbs -= I->Prob;
9125 case CC_JumpTable: {
9126 // FIXME: Optimize away range check based on pivot comparisons.
9127 JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
9128 JumpTable *JT = &JTCases[I->JTCasesIndex].second;
9130 // The jump block hasn't been inserted yet; insert it here.
9131 MachineBasicBlock *JumpMBB = JT->MBB;
9132 CurMF->insert(BBI, JumpMBB);
9134 auto JumpProb = I->Prob;
9135 auto FallthroughProb = UnhandledProbs;
9137 // If the default statement is a target of the jump table, we evenly
9138 // distribute the default probability to successors of CurMBB. Also
9139 // update the probability on the edge from JumpMBB to Fallthrough.
9140 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
9141 SE = JumpMBB->succ_end();
9143 if (*SI == DefaultMBB) {
9144 JumpProb += DefaultProb / 2;
9145 FallthroughProb -= DefaultProb / 2;
9146 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
9147 JumpMBB->normalizeSuccProbs();
9152 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
9153 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
9154 CurMBB->normalizeSuccProbs();
9156 // The jump table header will be inserted in our current block, do the
9157 // range check, and fall through to our fallthrough block.
9158 JTH->HeaderBB = CurMBB;
9159 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
9161 // If we're in the right place, emit the jump table header right now.
9162 if (CurMBB == SwitchMBB) {
9163 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
9164 JTH->Emitted = true;
9169 // FIXME: Optimize away range check based on pivot comparisons.
9170 BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
9172 // The bit test blocks haven't been inserted yet; insert them here.
9173 for (BitTestCase &BTC : BTB->Cases)
9174 CurMF->insert(BBI, BTC.ThisBB);
9176 // Fill in fields of the BitTestBlock.
9177 BTB->Parent = CurMBB;
9178 BTB->Default = Fallthrough;
9180 BTB->DefaultProb = UnhandledProbs;
9181 // If the cases in bit test don't form a contiguous range, we evenly
9182 // distribute the probability on the edge to Fallthrough to two
9183 // successors of CurMBB.
9184 if (!BTB->ContiguousRange) {
9185 BTB->Prob += DefaultProb / 2;
9186 BTB->DefaultProb -= DefaultProb / 2;
9189 // If we're in the right place, emit the bit test header right now.
9190 if (CurMBB == SwitchMBB) {
9191 visitBitTestHeader(*BTB, SwitchMBB);
9192 BTB->Emitted = true;
9197 const Value *RHS, *LHS, *MHS;
9199 if (I->Low == I->High) {
9200 // Check Cond == I->Low.
9206 // Check I->Low <= Cond <= I->High.
9213 // The false probability is the sum of all unhandled cases.
9214 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Prob,
9217 if (CurMBB == SwitchMBB)
9218 visitSwitchCase(CB, SwitchMBB);
9220 SwitchCases.push_back(CB);
9225 CurMBB = Fallthrough;
9229 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
9230 CaseClusterIt First,
9231 CaseClusterIt Last) {
9232 return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
9233 if (X.Prob != CC.Prob)
9234 return X.Prob > CC.Prob;
9236 // Ties are broken by comparing the case value.
9237 return X.Low->getValue().slt(CC.Low->getValue());
9241 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
9242 const SwitchWorkListItem &W,
9244 MachineBasicBlock *SwitchMBB) {
9245 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
9246 "Clusters not sorted?");
9248 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
9250 // Balance the tree based on branch probabilities to create a near-optimal (in
9251 // terms of search time given key frequency) binary search tree. See e.g. Kurt
9252 // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
9253 CaseClusterIt LastLeft = W.FirstCluster;
9254 CaseClusterIt FirstRight = W.LastCluster;
9255 auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
9256 auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
9258 // Move LastLeft and FirstRight towards each other from opposite directions to
9259 // find a partitioning of the clusters which balances the probability on both
9260 // sides. If LeftProb and RightProb are equal, alternate which side is
9261 // taken to ensure 0-probability nodes are distributed evenly.
9263 while (LastLeft + 1 < FirstRight) {
9264 if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
9265 LeftProb += (++LastLeft)->Prob;
9267 RightProb += (--FirstRight)->Prob;
9272 // Our binary search tree differs from a typical BST in that ours can have up
9273 // to three values in each leaf. The pivot selection above doesn't take that
9274 // into account, which means the tree might require more nodes and be less
9275 // efficient. We compensate for this here.
9277 unsigned NumLeft = LastLeft - W.FirstCluster + 1;
9278 unsigned NumRight = W.LastCluster - FirstRight + 1;
9280 if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
9281 // If one side has less than 3 clusters, and the other has more than 3,
9282 // consider taking a cluster from the other side.
9284 if (NumLeft < NumRight) {
9285 // Consider moving the first cluster on the right to the left side.
9286 CaseCluster &CC = *FirstRight;
9287 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
9288 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
9289 if (LeftSideRank <= RightSideRank) {
9290 // Moving the cluster to the left does not demote it.
9296 assert(NumRight < NumLeft);
9297 // Consider moving the last element on the left to the right side.
9298 CaseCluster &CC = *LastLeft;
9299 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
9300 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
9301 if (RightSideRank <= LeftSideRank) {
9302 // Moving the cluster to the right does not demot it.
9312 assert(LastLeft + 1 == FirstRight);
9313 assert(LastLeft >= W.FirstCluster);
9314 assert(FirstRight <= W.LastCluster);
9316 // Use the first element on the right as pivot since we will make less-than
9317 // comparisons against it.
9318 CaseClusterIt PivotCluster = FirstRight;
9319 assert(PivotCluster > W.FirstCluster);
9320 assert(PivotCluster <= W.LastCluster);
9322 CaseClusterIt FirstLeft = W.FirstCluster;
9323 CaseClusterIt LastRight = W.LastCluster;
9325 const ConstantInt *Pivot = PivotCluster->Low;
9327 // New blocks will be inserted immediately after the current one.
9328 MachineFunction::iterator BBI(W.MBB);
9331 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
9332 // we can branch to its destination directly if it's squeezed exactly in
9333 // between the known lower bound and Pivot - 1.
9334 MachineBasicBlock *LeftMBB;
9335 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
9336 FirstLeft->Low == W.GE &&
9337 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
9338 LeftMBB = FirstLeft->MBB;
9340 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
9341 FuncInfo.MF->insert(BBI, LeftMBB);
9343 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
9344 // Put Cond in a virtual register to make it available from the new blocks.
9345 ExportFromCurrentBlock(Cond);
9348 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
9349 // single cluster, RHS.Low == Pivot, and we can branch to its destination
9350 // directly if RHS.High equals the current upper bound.
9351 MachineBasicBlock *RightMBB;
9352 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
9353 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
9354 RightMBB = FirstRight->MBB;
9356 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
9357 FuncInfo.MF->insert(BBI, RightMBB);
9359 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
9360 // Put Cond in a virtual register to make it available from the new blocks.
9361 ExportFromCurrentBlock(Cond);
9364 // Create the CaseBlock record that will be used to lower the branch.
9365 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
9366 LeftProb, RightProb);
9368 if (W.MBB == SwitchMBB)
9369 visitSwitchCase(CB, SwitchMBB);
9371 SwitchCases.push_back(CB);
9374 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
9375 // Extract cases from the switch.
9376 BranchProbabilityInfo *BPI = FuncInfo.BPI;
9377 CaseClusterVector Clusters;
9378 Clusters.reserve(SI.getNumCases());
9379 for (auto I : SI.cases()) {
9380 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
9381 const ConstantInt *CaseVal = I.getCaseValue();
9382 BranchProbability Prob =
9383 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
9384 : BranchProbability(1, SI.getNumCases() + 1);
9385 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
9388 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
9390 // Cluster adjacent cases with the same destination. We do this at all
9391 // optimization levels because it's cheap to do and will make codegen faster
9392 // if there are many clusters.
9393 sortAndRangeify(Clusters);
9395 if (TM.getOptLevel() != CodeGenOpt::None) {
9396 // Replace an unreachable default with the most popular destination.
9397 // FIXME: Exploit unreachable default more aggressively.
9398 bool UnreachableDefault =
9399 isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
9400 if (UnreachableDefault && !Clusters.empty()) {
9401 DenseMap<const BasicBlock *, unsigned> Popularity;
9402 unsigned MaxPop = 0;
9403 const BasicBlock *MaxBB = nullptr;
9404 for (auto I : SI.cases()) {
9405 const BasicBlock *BB = I.getCaseSuccessor();
9406 if (++Popularity[BB] > MaxPop) {
9407 MaxPop = Popularity[BB];
9412 assert(MaxPop > 0 && MaxBB);
9413 DefaultMBB = FuncInfo.MBBMap[MaxBB];
9415 // Remove cases that were pointing to the destination that is now the
9417 CaseClusterVector New;
9418 New.reserve(Clusters.size());
9419 for (CaseCluster &CC : Clusters) {
9420 if (CC.MBB != DefaultMBB)
9423 Clusters = std::move(New);
9427 // If there is only the default destination, jump there directly.
9428 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
9429 if (Clusters.empty()) {
9430 SwitchMBB->addSuccessor(DefaultMBB);
9431 if (DefaultMBB != NextBlock(SwitchMBB)) {
9432 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
9433 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
9438 findJumpTables(Clusters, &SI, DefaultMBB);
9439 findBitTestClusters(Clusters, &SI);
9442 dbgs() << "Case clusters: ";
9443 for (const CaseCluster &C : Clusters) {
9444 if (C.Kind == CC_JumpTable) dbgs() << "JT:";
9445 if (C.Kind == CC_BitTests) dbgs() << "BT:";
9447 C.Low->getValue().print(dbgs(), true);
9448 if (C.Low != C.High) {
9450 C.High->getValue().print(dbgs(), true);
9457 assert(!Clusters.empty());
9458 SwitchWorkList WorkList;
9459 CaseClusterIt First = Clusters.begin();
9460 CaseClusterIt Last = Clusters.end() - 1;
9461 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
9462 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
9464 while (!WorkList.empty()) {
9465 SwitchWorkListItem W = WorkList.back();
9466 WorkList.pop_back();
9467 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
9469 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
9470 !DefaultMBB->getParent()->getFunction()->optForMinSize()) {
9471 // For optimized builds, lower large range as a balanced binary tree.
9472 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
9476 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);