1 //===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This implements routines for translating from LLVM IR into SelectionDAG IR.
12 //===----------------------------------------------------------------------===//
14 #include "SelectionDAGBuilder.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/Analysis/AliasAnalysis.h"
21 #include "llvm/Analysis/BranchProbabilityInfo.h"
22 #include "llvm/Analysis/ConstantFolding.h"
23 #include "llvm/Analysis/Loads.h"
24 #include "llvm/Analysis/TargetLibraryInfo.h"
25 #include "llvm/Analysis/ValueTracking.h"
26 #include "llvm/Analysis/VectorUtils.h"
27 #include "llvm/CodeGen/Analysis.h"
28 #include "llvm/CodeGen/FastISel.h"
29 #include "llvm/CodeGen/FunctionLoweringInfo.h"
30 #include "llvm/CodeGen/GCMetadata.h"
31 #include "llvm/CodeGen/GCStrategy.h"
32 #include "llvm/CodeGen/MachineFrameInfo.h"
33 #include "llvm/CodeGen/MachineFunction.h"
34 #include "llvm/CodeGen/MachineInstrBuilder.h"
35 #include "llvm/CodeGen/MachineJumpTableInfo.h"
36 #include "llvm/CodeGen/MachineModuleInfo.h"
37 #include "llvm/CodeGen/MachineRegisterInfo.h"
38 #include "llvm/CodeGen/SelectionDAG.h"
39 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
40 #include "llvm/CodeGen/StackMaps.h"
41 #include "llvm/CodeGen/WinEHFuncInfo.h"
42 #include "llvm/IR/CallingConv.h"
43 #include "llvm/IR/ConstantRange.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DataLayout.h"
46 #include "llvm/IR/DebugInfo.h"
47 #include "llvm/IR/DerivedTypes.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/GetElementPtrTypeIterator.h"
50 #include "llvm/IR/GlobalVariable.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/Instructions.h"
53 #include "llvm/IR/IntrinsicInst.h"
54 #include "llvm/IR/Intrinsics.h"
55 #include "llvm/IR/LLVMContext.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/Statepoint.h"
58 #include "llvm/MC/MCSymbol.h"
59 #include "llvm/Support/CommandLine.h"
60 #include "llvm/Support/Debug.h"
61 #include "llvm/Support/ErrorHandling.h"
62 #include "llvm/Support/MathExtras.h"
63 #include "llvm/Support/raw_ostream.h"
64 #include "llvm/Target/TargetFrameLowering.h"
65 #include "llvm/Target/TargetInstrInfo.h"
66 #include "llvm/Target/TargetIntrinsicInfo.h"
67 #include "llvm/Target/TargetLowering.h"
68 #include "llvm/Target/TargetOptions.h"
69 #include "llvm/Target/TargetSubtargetInfo.h"
74 #define DEBUG_TYPE "isel"
76 /// LimitFloatPrecision - Generate low-precision inline sequences for
77 /// some float libcalls (6, 8 or 12 bits).
78 static unsigned LimitFloatPrecision;
80 static cl::opt<unsigned, true>
81 LimitFPPrecision("limit-float-precision",
82 cl::desc("Generate low-precision inline sequences "
83 "for some float libcalls"),
84 cl::location(LimitFloatPrecision),
86 // Limit the width of DAG chains. This is important in general to prevent
87 // DAG-based analysis from blowing up. For example, alias analysis and
88 // load clustering may not complete in reasonable time. It is difficult to
89 // recognize and avoid this situation within each individual analysis, and
90 // future analyses are likely to have the same behavior. Limiting DAG width is
91 // the safe approach and will be especially important with global DAGs.
93 // MaxParallelChains default is arbitrarily high to avoid affecting
94 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
95 // sequence over this should have been converted to llvm.memcpy by the
96 // frontend. It is easy to induce this behavior with .ll code such as:
97 // %buffer = alloca [4096 x i8]
98 // %data = load [4096 x i8]* %argPtr
99 // store [4096 x i8] %data, [4096 x i8]* %buffer
100 static const unsigned MaxParallelChains = 64;
102 // True if the Value passed requires ABI mangling as it is a parameter to a
103 // function or a return value from a function which is not an intrinsic.
104 static bool isABIRegCopy(const Value * V) {
105 const bool IsRetInst = V && isa<ReturnInst>(V);
106 const bool IsCallInst = V && isa<CallInst>(V);
107 const bool IsInLineAsm =
108 IsCallInst && static_cast<const CallInst *>(V)->isInlineAsm();
109 const bool IsIndirectFunctionCall =
110 IsCallInst && !IsInLineAsm &&
111 !static_cast<const CallInst *>(V)->getCalledFunction();
112 // It is possible that the call instruction is an inline asm statement or an
113 // indirect function call in which case the return value of
114 // getCalledFunction() would be nullptr.
115 const bool IsInstrinsicCall =
116 IsCallInst && !IsInLineAsm && !IsIndirectFunctionCall &&
117 static_cast<const CallInst *>(V)->getCalledFunction()->getIntrinsicID() !=
118 Intrinsic::not_intrinsic;
120 return IsRetInst || (IsCallInst && (!IsInLineAsm && !IsInstrinsicCall));
123 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
124 const SDValue *Parts, unsigned NumParts,
125 MVT PartVT, EVT ValueVT, const Value *V,
128 /// getCopyFromParts - Create a value that contains the specified legal parts
129 /// combined into the value they represent. If the parts combine to a type
130 /// larger than ValueVT then AssertOp can be used to specify whether the extra
131 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
132 /// (ISD::AssertSext).
133 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
134 const SDValue *Parts, unsigned NumParts,
135 MVT PartVT, EVT ValueVT, const Value *V,
136 Optional<ISD::NodeType> AssertOp = None,
137 bool IsABIRegCopy = false) {
138 if (ValueVT.isVector())
139 return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
140 PartVT, ValueVT, V, IsABIRegCopy);
142 assert(NumParts > 0 && "No parts to assemble!");
143 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
144 SDValue Val = Parts[0];
147 // Assemble the value from multiple parts.
148 if (ValueVT.isInteger()) {
149 unsigned PartBits = PartVT.getSizeInBits();
150 unsigned ValueBits = ValueVT.getSizeInBits();
152 // Assemble the power of 2 part.
153 unsigned RoundParts = NumParts & (NumParts - 1) ?
154 1 << Log2_32(NumParts) : NumParts;
155 unsigned RoundBits = PartBits * RoundParts;
156 EVT RoundVT = RoundBits == ValueBits ?
157 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
160 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
162 if (RoundParts > 2) {
163 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
165 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
166 RoundParts / 2, PartVT, HalfVT, V);
168 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
169 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
172 if (DAG.getDataLayout().isBigEndian())
175 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
177 if (RoundParts < NumParts) {
178 // Assemble the trailing non-power-of-2 part.
179 unsigned OddParts = NumParts - RoundParts;
180 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
181 Hi = getCopyFromParts(DAG, DL,
182 Parts + RoundParts, OddParts, PartVT, OddVT, V);
184 // Combine the round and odd parts.
186 if (DAG.getDataLayout().isBigEndian())
188 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
189 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
191 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
192 DAG.getConstant(Lo.getValueSizeInBits(), DL,
193 TLI.getPointerTy(DAG.getDataLayout())));
194 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
195 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
197 } else if (PartVT.isFloatingPoint()) {
198 // FP split into multiple FP parts (for ppcf128)
199 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
202 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
203 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
204 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
206 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
208 // FP split into integer parts (soft fp)
209 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
210 !PartVT.isVector() && "Unexpected split");
211 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
212 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
216 // There is now one part, held in Val. Correct it to match ValueVT.
217 // PartEVT is the type of the register class that holds the value.
218 // ValueVT is the type of the inline asm operation.
219 EVT PartEVT = Val.getValueType();
221 if (PartEVT == ValueVT)
224 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
225 ValueVT.bitsLT(PartEVT)) {
226 // For an FP value in an integer part, we need to truncate to the right
228 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
229 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
232 // Handle types that have the same size.
233 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
234 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
236 // Handle types with different sizes.
237 if (PartEVT.isInteger() && ValueVT.isInteger()) {
238 if (ValueVT.bitsLT(PartEVT)) {
239 // For a truncate, see if we have any information to
240 // indicate whether the truncated bits will always be
241 // zero or sign-extension.
242 if (AssertOp.hasValue())
243 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
244 DAG.getValueType(ValueVT));
245 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
247 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
250 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
251 // FP_ROUND's are always exact here.
252 if (ValueVT.bitsLT(Val.getValueType()))
254 ISD::FP_ROUND, DL, ValueVT, Val,
255 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
257 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
260 llvm_unreachable("Unknown mismatch!");
263 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
264 const Twine &ErrMsg) {
265 const Instruction *I = dyn_cast_or_null<Instruction>(V);
267 return Ctx.emitError(ErrMsg);
269 const char *AsmError = ", possible invalid constraint for vector type";
270 if (const CallInst *CI = dyn_cast<CallInst>(I))
271 if (isa<InlineAsm>(CI->getCalledValue()))
272 return Ctx.emitError(I, ErrMsg + AsmError);
274 return Ctx.emitError(I, ErrMsg);
277 /// getCopyFromPartsVector - Create a value that contains the specified legal
278 /// parts combined into the value they represent. If the parts combine to a
279 /// type larger than ValueVT then AssertOp can be used to specify whether the
280 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
281 /// ValueVT (ISD::AssertSext).
282 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
283 const SDValue *Parts, unsigned NumParts,
284 MVT PartVT, EVT ValueVT, const Value *V,
286 assert(ValueVT.isVector() && "Not a vector value");
287 assert(NumParts > 0 && "No parts to assemble!");
288 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
289 SDValue Val = Parts[0];
291 // Handle a multi-element vector.
295 unsigned NumIntermediates;
299 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
300 *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates,
304 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
305 NumIntermediates, RegisterVT);
308 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
309 NumParts = NumRegs; // Silence a compiler warning.
310 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
311 assert(RegisterVT.getSizeInBits() ==
312 Parts[0].getSimpleValueType().getSizeInBits() &&
313 "Part type sizes don't match!");
315 // Assemble the parts into intermediate operands.
316 SmallVector<SDValue, 8> Ops(NumIntermediates);
317 if (NumIntermediates == NumParts) {
318 // If the register was not expanded, truncate or copy the value,
320 for (unsigned i = 0; i != NumParts; ++i)
321 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
322 PartVT, IntermediateVT, V);
323 } else if (NumParts > 0) {
324 // If the intermediate type was expanded, build the intermediate
325 // operands from the parts.
326 assert(NumParts % NumIntermediates == 0 &&
327 "Must expand into a divisible number of parts!");
328 unsigned Factor = NumParts / NumIntermediates;
329 for (unsigned i = 0; i != NumIntermediates; ++i)
330 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
331 PartVT, IntermediateVT, V);
334 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
335 // intermediate operands.
337 EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
338 (IntermediateVT.isVector()
339 ? IntermediateVT.getVectorNumElements() * NumParts
340 : NumIntermediates));
341 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
343 DL, BuiltVectorTy, Ops);
346 // There is now one part, held in Val. Correct it to match ValueVT.
347 EVT PartEVT = Val.getValueType();
349 if (PartEVT == ValueVT)
352 if (PartEVT.isVector()) {
353 // If the element type of the source/dest vectors are the same, but the
354 // parts vector has more elements than the value vector, then we have a
355 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
357 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
358 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
359 "Cannot narrow, it would be a lossy transformation");
361 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
362 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
365 // Vector/Vector bitcast.
366 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
367 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
369 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
370 "Cannot handle this kind of promotion");
371 // Promoted vector extract
372 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
376 // Trivial bitcast if the types are the same size and the destination
377 // vector type is legal.
378 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
379 TLI.isTypeLegal(ValueVT))
380 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
382 if (ValueVT.getVectorNumElements() != 1) {
383 // Certain ABIs require that vectors are passed as integers. For vectors
384 // are the same size, this is an obvious bitcast.
385 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
386 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
387 } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
388 // Bitcast Val back the original type and extract the corresponding
390 unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
391 EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
392 ValueVT.getVectorElementType(), Elts);
393 Val = DAG.getBitcast(WiderVecType, Val);
395 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
396 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
399 diagnosePossiblyInvalidConstraint(
400 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
401 return DAG.getUNDEF(ValueVT);
404 // Handle cases such as i8 -> <1 x i1>
405 EVT ValueSVT = ValueVT.getVectorElementType();
406 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
407 Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
408 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
410 return DAG.getBuildVector(ValueVT, DL, Val);
413 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
414 SDValue Val, SDValue *Parts, unsigned NumParts,
415 MVT PartVT, const Value *V, bool IsABIRegCopy);
417 /// getCopyToParts - Create a series of nodes that contain the specified value
418 /// split into legal parts. If the parts contain more bits than Val, then, for
419 /// integers, ExtendKind can be used to specify how to generate the extra bits.
420 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
421 SDValue *Parts, unsigned NumParts, MVT PartVT,
423 ISD::NodeType ExtendKind = ISD::ANY_EXTEND,
424 bool IsABIRegCopy = false) {
425 EVT ValueVT = Val.getValueType();
427 // Handle the vector case separately.
428 if (ValueVT.isVector())
429 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
432 unsigned PartBits = PartVT.getSizeInBits();
433 unsigned OrigNumParts = NumParts;
434 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
435 "Copying to an illegal type!");
440 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
441 EVT PartEVT = PartVT;
442 if (PartEVT == ValueVT) {
443 assert(NumParts == 1 && "No-op copy with multiple parts!");
448 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
449 // If the parts cover more bits than the value has, promote the value.
450 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
451 assert(NumParts == 1 && "Do not know what to promote to!");
452 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
454 if (ValueVT.isFloatingPoint()) {
455 // FP values need to be bitcast, then extended if they are being put
456 // into a larger container.
457 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
458 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
460 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
461 ValueVT.isInteger() &&
462 "Unknown mismatch!");
463 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
464 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
465 if (PartVT == MVT::x86mmx)
466 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
468 } else if (PartBits == ValueVT.getSizeInBits()) {
469 // Different types of the same size.
470 assert(NumParts == 1 && PartEVT != ValueVT);
471 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
472 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
473 // If the parts cover less bits than value has, truncate the value.
474 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
475 ValueVT.isInteger() &&
476 "Unknown mismatch!");
477 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
478 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
479 if (PartVT == MVT::x86mmx)
480 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
483 // The value may have changed - recompute ValueVT.
484 ValueVT = Val.getValueType();
485 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
486 "Failed to tile the value with PartVT!");
489 if (PartEVT != ValueVT) {
490 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
491 "scalar-to-vector conversion failed");
492 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
499 // Expand the value into multiple parts.
500 if (NumParts & (NumParts - 1)) {
501 // The number of parts is not a power of 2. Split off and copy the tail.
502 assert(PartVT.isInteger() && ValueVT.isInteger() &&
503 "Do not know what to expand to!");
504 unsigned RoundParts = 1 << Log2_32(NumParts);
505 unsigned RoundBits = RoundParts * PartBits;
506 unsigned OddParts = NumParts - RoundParts;
507 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
508 DAG.getIntPtrConstant(RoundBits, DL));
509 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
511 if (DAG.getDataLayout().isBigEndian())
512 // The odd parts were reversed by getCopyToParts - unreverse them.
513 std::reverse(Parts + RoundParts, Parts + NumParts);
515 NumParts = RoundParts;
516 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
517 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
520 // The number of parts is a power of 2. Repeatedly bisect the value using
522 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
523 EVT::getIntegerVT(*DAG.getContext(),
524 ValueVT.getSizeInBits()),
527 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
528 for (unsigned i = 0; i < NumParts; i += StepSize) {
529 unsigned ThisBits = StepSize * PartBits / 2;
530 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
531 SDValue &Part0 = Parts[i];
532 SDValue &Part1 = Parts[i+StepSize/2];
534 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
535 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
536 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
537 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
539 if (ThisBits == PartBits && ThisVT != PartVT) {
540 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
541 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
546 if (DAG.getDataLayout().isBigEndian())
547 std::reverse(Parts, Parts + OrigNumParts);
551 /// getCopyToPartsVector - Create a series of nodes that contain the specified
552 /// value split into legal parts.
553 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
554 SDValue Val, SDValue *Parts, unsigned NumParts,
555 MVT PartVT, const Value *V,
558 EVT ValueVT = Val.getValueType();
559 assert(ValueVT.isVector() && "Not a vector");
560 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
563 EVT PartEVT = PartVT;
564 if (PartEVT == ValueVT) {
566 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
567 // Bitconvert vector->vector case.
568 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
569 } else if (PartVT.isVector() &&
570 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
571 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
572 EVT ElementVT = PartVT.getVectorElementType();
573 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
575 SmallVector<SDValue, 16> Ops;
576 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
577 Ops.push_back(DAG.getNode(
578 ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
579 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
581 for (unsigned i = ValueVT.getVectorNumElements(),
582 e = PartVT.getVectorNumElements(); i != e; ++i)
583 Ops.push_back(DAG.getUNDEF(ElementVT));
585 Val = DAG.getBuildVector(PartVT, DL, Ops);
587 // FIXME: Use CONCAT for 2x -> 4x.
589 //SDValue UndefElts = DAG.getUNDEF(VectorTy);
590 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
591 } else if (PartVT.isVector() &&
592 PartEVT.getVectorElementType().bitsGE(
593 ValueVT.getVectorElementType()) &&
594 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
596 // Promoted vector extract
597 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
599 if (ValueVT.getVectorNumElements() == 1) {
601 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
602 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
605 assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
606 "lossy conversion of vector to scalar type");
607 EVT IntermediateType =
608 EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
609 Val = DAG.getBitcast(IntermediateType, Val);
610 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
614 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
619 // Handle a multi-element vector.
622 unsigned NumIntermediates;
625 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
626 *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates,
630 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
631 NumIntermediates, RegisterVT);
633 unsigned NumElements = ValueVT.getVectorNumElements();
635 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
636 NumParts = NumRegs; // Silence a compiler warning.
637 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
639 // Convert the vector to the appropiate type if necessary.
640 unsigned DestVectorNoElts =
642 (IntermediateVT.isVector() ? IntermediateVT.getVectorNumElements() : 1);
643 EVT BuiltVectorTy = EVT::getVectorVT(
644 *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
645 if (Val.getValueType() != BuiltVectorTy)
646 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
648 // Split the vector into intermediate operands.
649 SmallVector<SDValue, 8> Ops(NumIntermediates);
650 for (unsigned i = 0; i != NumIntermediates; ++i) {
651 if (IntermediateVT.isVector())
653 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
654 DAG.getConstant(i * (NumElements / NumIntermediates), DL,
655 TLI.getVectorIdxTy(DAG.getDataLayout())));
657 Ops[i] = DAG.getNode(
658 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
659 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
662 // Split the intermediate operands into legal parts.
663 if (NumParts == NumIntermediates) {
664 // If the register was not expanded, promote or copy the value,
666 for (unsigned i = 0; i != NumParts; ++i)
667 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
668 } else if (NumParts > 0) {
669 // If the intermediate type was expanded, split each the value into
671 assert(NumIntermediates != 0 && "division by zero");
672 assert(NumParts % NumIntermediates == 0 &&
673 "Must expand into a divisible number of parts!");
674 unsigned Factor = NumParts / NumIntermediates;
675 for (unsigned i = 0; i != NumIntermediates; ++i)
676 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
680 RegsForValue::RegsForValue() { IsABIMangled = false; }
682 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt,
683 EVT valuevt, bool IsABIMangledValue)
684 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
685 RegCount(1, regs.size()), IsABIMangled(IsABIMangledValue) {}
687 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
688 const DataLayout &DL, unsigned Reg, Type *Ty,
689 bool IsABIMangledValue) {
690 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
692 IsABIMangled = IsABIMangledValue;
694 for (EVT ValueVT : ValueVTs) {
695 unsigned NumRegs = IsABIMangledValue
696 ? TLI.getNumRegistersForCallingConv(Context, ValueVT)
697 : TLI.getNumRegisters(Context, ValueVT);
698 MVT RegisterVT = IsABIMangledValue
699 ? TLI.getRegisterTypeForCallingConv(Context, ValueVT)
700 : TLI.getRegisterType(Context, ValueVT);
701 for (unsigned i = 0; i != NumRegs; ++i)
702 Regs.push_back(Reg + i);
703 RegVTs.push_back(RegisterVT);
704 RegCount.push_back(NumRegs);
709 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
710 FunctionLoweringInfo &FuncInfo,
711 const SDLoc &dl, SDValue &Chain,
712 SDValue *Flag, const Value *V) const {
713 // A Value with type {} or [0 x %t] needs no registers.
714 if (ValueVTs.empty())
717 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
719 // Assemble the legal parts into the final values.
720 SmallVector<SDValue, 4> Values(ValueVTs.size());
721 SmallVector<SDValue, 8> Parts;
722 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
723 // Copy the legal parts from the registers.
724 EVT ValueVT = ValueVTs[Value];
725 unsigned NumRegs = RegCount[Value];
726 MVT RegisterVT = IsABIMangled
727 ? TLI.getRegisterTypeForCallingConv(RegVTs[Value])
730 Parts.resize(NumRegs);
731 for (unsigned i = 0; i != NumRegs; ++i) {
734 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
736 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
737 *Flag = P.getValue(2);
740 Chain = P.getValue(1);
743 // If the source register was virtual and if we know something about it,
744 // add an assert node.
745 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
746 !RegisterVT.isInteger() || RegisterVT.isVector())
749 const FunctionLoweringInfo::LiveOutInfo *LOI =
750 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
754 unsigned RegSize = RegisterVT.getSizeInBits();
755 unsigned NumSignBits = LOI->NumSignBits;
756 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
758 if (NumZeroBits == RegSize) {
759 // The current value is a zero.
760 // Explicitly express that as it would be easier for
761 // optimizations to kick in.
762 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
766 // FIXME: We capture more information than the dag can represent. For
767 // now, just use the tightest assertzext/assertsext possible.
769 EVT FromVT(MVT::Other);
770 if (NumSignBits == RegSize) {
771 isSExt = true; // ASSERT SEXT 1
773 } else if (NumZeroBits >= RegSize - 1) {
774 isSExt = false; // ASSERT ZEXT 1
776 } else if (NumSignBits > RegSize - 8) {
777 isSExt = true; // ASSERT SEXT 8
779 } else if (NumZeroBits >= RegSize - 8) {
780 isSExt = false; // ASSERT ZEXT 8
782 } else if (NumSignBits > RegSize - 16) {
783 isSExt = true; // ASSERT SEXT 16
785 } else if (NumZeroBits >= RegSize - 16) {
786 isSExt = false; // ASSERT ZEXT 16
788 } else if (NumSignBits > RegSize - 32) {
789 isSExt = true; // ASSERT SEXT 32
791 } else if (NumZeroBits >= RegSize - 32) {
792 isSExt = false; // ASSERT ZEXT 32
797 // Add an assertion node.
798 assert(FromVT != MVT::Other);
799 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
800 RegisterVT, P, DAG.getValueType(FromVT));
803 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
804 NumRegs, RegisterVT, ValueVT, V);
809 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
812 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
813 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
815 ISD::NodeType PreferredExtendType) const {
816 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
817 ISD::NodeType ExtendKind = PreferredExtendType;
819 // Get the list of the values's legal parts.
820 unsigned NumRegs = Regs.size();
821 SmallVector<SDValue, 8> Parts(NumRegs);
822 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
823 unsigned NumParts = RegCount[Value];
825 MVT RegisterVT = IsABIMangled
826 ? TLI.getRegisterTypeForCallingConv(RegVTs[Value])
829 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
830 ExtendKind = ISD::ZERO_EXTEND;
832 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
833 &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
837 // Copy the parts into the registers.
838 SmallVector<SDValue, 8> Chains(NumRegs);
839 for (unsigned i = 0; i != NumRegs; ++i) {
842 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
844 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
845 *Flag = Part.getValue(1);
848 Chains[i] = Part.getValue(0);
851 if (NumRegs == 1 || Flag)
852 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
853 // flagged to it. That is the CopyToReg nodes and the user are considered
854 // a single scheduling unit. If we create a TokenFactor and return it as
855 // chain, then the TokenFactor is both a predecessor (operand) of the
856 // user as well as a successor (the TF operands are flagged to the user).
857 // c1, f1 = CopyToReg
858 // c2, f2 = CopyToReg
859 // c3 = TokenFactor c1, c2
862 Chain = Chains[NumRegs-1];
864 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
867 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
868 unsigned MatchingIdx, const SDLoc &dl,
870 std::vector<SDValue> &Ops) const {
871 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
873 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
875 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
876 else if (!Regs.empty() &&
877 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
878 // Put the register class of the virtual registers in the flag word. That
879 // way, later passes can recompute register class constraints for inline
880 // assembly as well as normal instructions.
881 // Don't do this for tied operands that can use the regclass information
883 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
884 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
885 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
888 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
891 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
892 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
893 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
894 MVT RegisterVT = RegVTs[Value];
895 for (unsigned i = 0; i != NumRegs; ++i) {
896 assert(Reg < Regs.size() && "Mismatch in # registers expected");
897 unsigned TheReg = Regs[Reg++];
898 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
900 if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
901 // If we clobbered the stack pointer, MFI should know about it.
902 assert(DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment());
908 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
909 const TargetLibraryInfo *li) {
913 DL = &DAG.getDataLayout();
914 Context = DAG.getContext();
915 LPadToCallSiteMap.clear();
918 void SelectionDAGBuilder::clear() {
920 UnusedArgNodeMap.clear();
921 PendingLoads.clear();
922 PendingExports.clear();
925 SDNodeOrder = LowestSDNodeOrder;
926 StatepointLowering.clear();
929 void SelectionDAGBuilder::clearDanglingDebugInfo() {
930 DanglingDebugInfoMap.clear();
933 SDValue SelectionDAGBuilder::getRoot() {
934 if (PendingLoads.empty())
935 return DAG.getRoot();
937 if (PendingLoads.size() == 1) {
938 SDValue Root = PendingLoads[0];
940 PendingLoads.clear();
944 // Otherwise, we have to make a token factor node.
945 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
947 PendingLoads.clear();
952 SDValue SelectionDAGBuilder::getControlRoot() {
953 SDValue Root = DAG.getRoot();
955 if (PendingExports.empty())
958 // Turn all of the CopyToReg chains into one factored node.
959 if (Root.getOpcode() != ISD::EntryToken) {
960 unsigned i = 0, e = PendingExports.size();
961 for (; i != e; ++i) {
962 assert(PendingExports[i].getNode()->getNumOperands() > 1);
963 if (PendingExports[i].getNode()->getOperand(0) == Root)
964 break; // Don't add the root if we already indirectly depend on it.
968 PendingExports.push_back(Root);
971 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
973 PendingExports.clear();
978 void SelectionDAGBuilder::visit(const Instruction &I) {
979 // Set up outgoing PHI node register values before emitting the terminator.
980 if (isa<TerminatorInst>(&I)) {
981 HandlePHINodesInSuccessorBlocks(I.getParent());
984 // Increase the SDNodeOrder if dealing with a non-debug instruction.
985 if (!isa<DbgInfoIntrinsic>(I))
990 visit(I.getOpcode(), I);
992 if (!isa<TerminatorInst>(&I) && !HasTailCall &&
993 !isStatepoint(&I)) // statepoints handle their exports internally
994 CopyToExportRegsIfNeeded(&I);
999 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1000 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1003 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1004 // Note: this doesn't use InstVisitor, because it has to work with
1005 // ConstantExpr's in addition to instructions.
1007 default: llvm_unreachable("Unknown instruction type encountered!");
1008 // Build the switch statement using the Instruction.def file.
1009 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1010 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1011 #include "llvm/IR/Instruction.def"
1015 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1016 // generate the debug data structures now that we've seen its definition.
1017 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1019 DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
1021 const DbgValueInst *DI = DDI.getDI();
1022 DebugLoc dl = DDI.getdl();
1023 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1024 DILocalVariable *Variable = DI->getVariable();
1025 DIExpression *Expr = DI->getExpression();
1026 assert(Variable->isValidLocationForIntrinsic(dl) &&
1027 "Expected inlined-at fields to agree");
1028 uint64_t Offset = DI->getOffset();
1030 if (Val.getNode()) {
1031 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
1033 SDV = getDbgValue(Val, Variable, Expr, Offset, dl, DbgSDNodeOrder);
1034 DAG.AddDbgValue(SDV, Val.getNode(), false);
1037 DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1038 DanglingDebugInfoMap[V] = DanglingDebugInfo();
1042 /// getCopyFromRegs - If there was virtual register allocated for the value V
1043 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1044 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1045 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1048 if (It != FuncInfo.ValueMap.end()) {
1049 unsigned InReg = It->second;
1051 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1052 DAG.getDataLayout(), InReg, Ty, isABIRegCopy(V));
1053 SDValue Chain = DAG.getEntryNode();
1054 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1056 resolveDanglingDebugInfo(V, Result);
1062 /// getValue - Return an SDValue for the given Value.
1063 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1064 // If we already have an SDValue for this value, use it. It's important
1065 // to do this first, so that we don't create a CopyFromReg if we already
1066 // have a regular SDValue.
1067 SDValue &N = NodeMap[V];
1068 if (N.getNode()) return N;
1070 // If there's a virtual register allocated and initialized for this
1072 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1075 // Otherwise create a new SDValue and remember it.
1076 SDValue Val = getValueImpl(V);
1078 resolveDanglingDebugInfo(V, Val);
1082 // Return true if SDValue exists for the given Value
1083 bool SelectionDAGBuilder::findValue(const Value *V) const {
1084 return (NodeMap.find(V) != NodeMap.end()) ||
1085 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1088 /// getNonRegisterValue - Return an SDValue for the given Value, but
1089 /// don't look in FuncInfo.ValueMap for a virtual register.
1090 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1091 // If we already have an SDValue for this value, use it.
1092 SDValue &N = NodeMap[V];
1094 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1095 // Remove the debug location from the node as the node is about to be used
1096 // in a location which may differ from the original debug location. This
1097 // is relevant to Constant and ConstantFP nodes because they can appear
1098 // as constant expressions inside PHI nodes.
1099 N->setDebugLoc(DebugLoc());
1104 // Otherwise create a new SDValue and remember it.
1105 SDValue Val = getValueImpl(V);
1107 resolveDanglingDebugInfo(V, Val);
1111 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1112 /// Create an SDValue for the given value.
1113 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1114 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1116 if (const Constant *C = dyn_cast<Constant>(V)) {
1117 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1119 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1120 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1122 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1123 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1125 if (isa<ConstantPointerNull>(C)) {
1126 unsigned AS = V->getType()->getPointerAddressSpace();
1127 return DAG.getConstant(0, getCurSDLoc(),
1128 TLI.getPointerTy(DAG.getDataLayout(), AS));
1131 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1132 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1134 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1135 return DAG.getUNDEF(VT);
1137 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1138 visit(CE->getOpcode(), *CE);
1139 SDValue N1 = NodeMap[V];
1140 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1144 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1145 SmallVector<SDValue, 4> Constants;
1146 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1148 SDNode *Val = getValue(*OI).getNode();
1149 // If the operand is an empty aggregate, there are no values.
1151 // Add each leaf value from the operand to the Constants list
1152 // to form a flattened list of all the values.
1153 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1154 Constants.push_back(SDValue(Val, i));
1157 return DAG.getMergeValues(Constants, getCurSDLoc());
1160 if (const ConstantDataSequential *CDS =
1161 dyn_cast<ConstantDataSequential>(C)) {
1162 SmallVector<SDValue, 4> Ops;
1163 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1164 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1165 // Add each leaf value from the operand to the Constants list
1166 // to form a flattened list of all the values.
1167 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1168 Ops.push_back(SDValue(Val, i));
1171 if (isa<ArrayType>(CDS->getType()))
1172 return DAG.getMergeValues(Ops, getCurSDLoc());
1173 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1176 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1177 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1178 "Unknown struct or array constant!");
1180 SmallVector<EVT, 4> ValueVTs;
1181 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1182 unsigned NumElts = ValueVTs.size();
1184 return SDValue(); // empty struct
1185 SmallVector<SDValue, 4> Constants(NumElts);
1186 for (unsigned i = 0; i != NumElts; ++i) {
1187 EVT EltVT = ValueVTs[i];
1188 if (isa<UndefValue>(C))
1189 Constants[i] = DAG.getUNDEF(EltVT);
1190 else if (EltVT.isFloatingPoint())
1191 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1193 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1196 return DAG.getMergeValues(Constants, getCurSDLoc());
1199 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1200 return DAG.getBlockAddress(BA, VT);
1202 VectorType *VecTy = cast<VectorType>(V->getType());
1203 unsigned NumElements = VecTy->getNumElements();
1205 // Now that we know the number and type of the elements, get that number of
1206 // elements into the Ops array based on what kind of constant it is.
1207 SmallVector<SDValue, 16> Ops;
1208 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1209 for (unsigned i = 0; i != NumElements; ++i)
1210 Ops.push_back(getValue(CV->getOperand(i)));
1212 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1214 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1217 if (EltVT.isFloatingPoint())
1218 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1220 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1221 Ops.assign(NumElements, Op);
1224 // Create a BUILD_VECTOR node.
1225 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1228 // If this is a static alloca, generate it as the frameindex instead of
1230 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1231 DenseMap<const AllocaInst*, int>::iterator SI =
1232 FuncInfo.StaticAllocaMap.find(AI);
1233 if (SI != FuncInfo.StaticAllocaMap.end())
1234 return DAG.getFrameIndex(SI->second,
1235 TLI.getFrameIndexTy(DAG.getDataLayout()));
1238 // If this is an instruction which fast-isel has deferred, select it now.
1239 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1240 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1242 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1243 Inst->getType(), isABIRegCopy(V));
1244 SDValue Chain = DAG.getEntryNode();
1245 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1248 llvm_unreachable("Can't get register for value!");
1251 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1252 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1253 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1254 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1255 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1256 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1257 if (IsMSVCCXX || IsCoreCLR)
1258 CatchPadMBB->setIsEHFuncletEntry();
1260 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot()));
1263 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1264 // Update machine-CFG edge.
1265 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1266 FuncInfo.MBB->addSuccessor(TargetMBB);
1268 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1269 bool IsSEH = isAsynchronousEHPersonality(Pers);
1271 // If this is not a fall-through branch or optimizations are switched off,
1273 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1274 TM.getOptLevel() == CodeGenOpt::None)
1275 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1276 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1280 // Figure out the funclet membership for the catchret's successor.
1281 // This will be used by the FuncletLayout pass to determine how to order the
1283 // A 'catchret' returns to the outer scope's color.
1284 Value *ParentPad = I.getCatchSwitchParentPad();
1285 const BasicBlock *SuccessorColor;
1286 if (isa<ConstantTokenNone>(ParentPad))
1287 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1289 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1290 assert(SuccessorColor && "No parent funclet for catchret!");
1291 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1292 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1294 // Create the terminator node.
1295 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1296 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1297 DAG.getBasicBlock(SuccessorColorMBB));
1301 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1302 // Don't emit any special code for the cleanuppad instruction. It just marks
1303 // the start of a funclet.
1304 FuncInfo.MBB->setIsEHFuncletEntry();
1305 FuncInfo.MBB->setIsCleanupFuncletEntry();
1308 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1309 /// many places it could ultimately go. In the IR, we have a single unwind
1310 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1311 /// This function skips over imaginary basic blocks that hold catchswitch
1312 /// instructions, and finds all the "real" machine
1313 /// basic block destinations. As those destinations may not be successors of
1314 /// EHPadBB, here we also calculate the edge probability to those destinations.
1315 /// The passed-in Prob is the edge probability to EHPadBB.
1316 static void findUnwindDestinations(
1317 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1318 BranchProbability Prob,
1319 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1321 EHPersonality Personality =
1322 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1323 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1324 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1327 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1328 BasicBlock *NewEHPadBB = nullptr;
1329 if (isa<LandingPadInst>(Pad)) {
1330 // Stop on landingpads. They are not funclets.
1331 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1333 } else if (isa<CleanupPadInst>(Pad)) {
1334 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1336 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1337 UnwindDests.back().first->setIsEHFuncletEntry();
1339 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1340 // Add the catchpad handlers to the possible destinations.
1341 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1342 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1343 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1344 if (IsMSVCCXX || IsCoreCLR)
1345 UnwindDests.back().first->setIsEHFuncletEntry();
1347 NewEHPadBB = CatchSwitch->getUnwindDest();
1352 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1353 if (BPI && NewEHPadBB)
1354 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1355 EHPadBB = NewEHPadBB;
1359 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1360 // Update successor info.
1361 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1362 auto UnwindDest = I.getUnwindDest();
1363 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1364 BranchProbability UnwindDestProb =
1366 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1367 : BranchProbability::getZero();
1368 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1369 for (auto &UnwindDest : UnwindDests) {
1370 UnwindDest.first->setIsEHPad();
1371 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1373 FuncInfo.MBB->normalizeSuccProbs();
1375 // Create the terminator node.
1377 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1381 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1382 report_fatal_error("visitCatchSwitch not yet implemented!");
1385 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1386 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1387 auto &DL = DAG.getDataLayout();
1388 SDValue Chain = getControlRoot();
1389 SmallVector<ISD::OutputArg, 8> Outs;
1390 SmallVector<SDValue, 8> OutVals;
1392 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1395 // %val = call <ty> @llvm.experimental.deoptimize()
1399 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1400 LowerDeoptimizingReturn();
1404 if (!FuncInfo.CanLowerReturn) {
1405 unsigned DemoteReg = FuncInfo.DemoteRegister;
1406 const Function *F = I.getParent()->getParent();
1408 // Emit a store of the return value through the virtual register.
1409 // Leave Outs empty so that LowerReturn won't try to load return
1410 // registers the usual way.
1411 SmallVector<EVT, 1> PtrValueVTs;
1412 ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()),
1415 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1416 DemoteReg, PtrValueVTs[0]);
1417 SDValue RetOp = getValue(I.getOperand(0));
1419 SmallVector<EVT, 4> ValueVTs;
1420 SmallVector<uint64_t, 4> Offsets;
1421 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1422 unsigned NumValues = ValueVTs.size();
1424 // An aggregate return value cannot wrap around the address space, so
1425 // offsets to its parts don't wrap either.
1427 Flags.setNoUnsignedWrap(true);
1429 SmallVector<SDValue, 4> Chains(NumValues);
1430 for (unsigned i = 0; i != NumValues; ++i) {
1431 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
1432 RetPtr.getValueType(), RetPtr,
1433 DAG.getIntPtrConstant(Offsets[i],
1436 Chains[i] = DAG.getStore(Chain, getCurSDLoc(),
1437 SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1438 // FIXME: better loc info would be nice.
1439 Add, MachinePointerInfo());
1442 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1443 MVT::Other, Chains);
1444 } else if (I.getNumOperands() != 0) {
1445 SmallVector<EVT, 4> ValueVTs;
1446 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1447 unsigned NumValues = ValueVTs.size();
1449 SDValue RetOp = getValue(I.getOperand(0));
1451 const Function *F = I.getParent()->getParent();
1453 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1454 if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1456 ExtendKind = ISD::SIGN_EXTEND;
1457 else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1459 ExtendKind = ISD::ZERO_EXTEND;
1461 LLVMContext &Context = F->getContext();
1462 bool RetInReg = F->getAttributes().hasAttribute(
1463 AttributeList::ReturnIndex, Attribute::InReg);
1465 for (unsigned j = 0; j != NumValues; ++j) {
1466 EVT VT = ValueVTs[j];
1468 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1469 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1471 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, VT);
1472 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, VT);
1473 SmallVector<SDValue, 4> Parts(NumParts);
1474 getCopyToParts(DAG, getCurSDLoc(),
1475 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1476 &Parts[0], NumParts, PartVT, &I, ExtendKind, true);
1478 // 'inreg' on function refers to return value
1479 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1483 // Propagate extension type if any
1484 if (ExtendKind == ISD::SIGN_EXTEND)
1486 else if (ExtendKind == ISD::ZERO_EXTEND)
1489 for (unsigned i = 0; i < NumParts; ++i) {
1490 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1491 VT, /*isfixed=*/true, 0, 0));
1492 OutVals.push_back(Parts[i]);
1498 // Push in swifterror virtual register as the last element of Outs. This makes
1499 // sure swifterror virtual register will be returned in the swifterror
1500 // physical register.
1501 const Function *F = I.getParent()->getParent();
1502 if (TLI.supportSwiftError() &&
1503 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1504 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument");
1505 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1506 Flags.setSwiftError();
1507 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1508 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1509 true /*isfixed*/, 1 /*origidx*/,
1511 // Create SDNode for the swifterror virtual register.
1513 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1514 &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1515 EVT(TLI.getPointerTy(DL))));
1518 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1519 CallingConv::ID CallConv =
1520 DAG.getMachineFunction().getFunction()->getCallingConv();
1521 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1522 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1524 // Verify that the target's LowerReturn behaved as expected.
1525 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1526 "LowerReturn didn't return a valid chain!");
1528 // Update the DAG with the new chain value resulting from return lowering.
1532 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
1533 /// created for it, emit nodes to copy the value into the virtual
1535 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1537 if (V->getType()->isEmptyTy())
1540 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1541 if (VMI != FuncInfo.ValueMap.end()) {
1542 assert(!V->use_empty() && "Unused value assigned virtual registers!");
1543 CopyValueToVirtualRegister(V, VMI->second);
1547 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
1548 /// the current basic block, add it to ValueMap now so that we'll get a
1550 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1551 // No need to export constants.
1552 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1554 // Already exported?
1555 if (FuncInfo.isExportedInst(V)) return;
1557 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1558 CopyValueToVirtualRegister(V, Reg);
1561 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1562 const BasicBlock *FromBB) {
1563 // The operands of the setcc have to be in this block. We don't know
1564 // how to export them from some other block.
1565 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1566 // Can export from current BB.
1567 if (VI->getParent() == FromBB)
1570 // Is already exported, noop.
1571 return FuncInfo.isExportedInst(V);
1574 // If this is an argument, we can export it if the BB is the entry block or
1575 // if it is already exported.
1576 if (isa<Argument>(V)) {
1577 if (FromBB == &FromBB->getParent()->getEntryBlock())
1580 // Otherwise, can only export this if it is already exported.
1581 return FuncInfo.isExportedInst(V);
1584 // Otherwise, constants can always be exported.
1588 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1590 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1591 const MachineBasicBlock *Dst) const {
1592 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1593 const BasicBlock *SrcBB = Src->getBasicBlock();
1594 const BasicBlock *DstBB = Dst->getBasicBlock();
1596 // If BPI is not available, set the default probability as 1 / N, where N is
1597 // the number of successors.
1598 auto SuccSize = std::max<uint32_t>(
1599 std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
1600 return BranchProbability(1, SuccSize);
1602 return BPI->getEdgeProbability(SrcBB, DstBB);
1605 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1606 MachineBasicBlock *Dst,
1607 BranchProbability Prob) {
1609 Src->addSuccessorWithoutProb(Dst);
1611 if (Prob.isUnknown())
1612 Prob = getEdgeProbability(Src, Dst);
1613 Src->addSuccessor(Dst, Prob);
1617 static bool InBlock(const Value *V, const BasicBlock *BB) {
1618 if (const Instruction *I = dyn_cast<Instruction>(V))
1619 return I->getParent() == BB;
1623 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1624 /// This function emits a branch and is used at the leaves of an OR or an
1625 /// AND operator tree.
1628 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1629 MachineBasicBlock *TBB,
1630 MachineBasicBlock *FBB,
1631 MachineBasicBlock *CurBB,
1632 MachineBasicBlock *SwitchBB,
1633 BranchProbability TProb,
1634 BranchProbability FProb,
1636 const BasicBlock *BB = CurBB->getBasicBlock();
1638 // If the leaf of the tree is a comparison, merge the condition into
1640 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1641 // The operands of the cmp have to be in this block. We don't know
1642 // how to export them from some other block. If this is the first block
1643 // of the sequence, no exporting is needed.
1644 if (CurBB == SwitchBB ||
1645 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1646 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1647 ISD::CondCode Condition;
1648 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1649 ICmpInst::Predicate Pred =
1650 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1651 Condition = getICmpCondCode(Pred);
1653 const FCmpInst *FC = cast<FCmpInst>(Cond);
1654 FCmpInst::Predicate Pred =
1655 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1656 Condition = getFCmpCondCode(Pred);
1657 if (TM.Options.NoNaNsFPMath)
1658 Condition = getFCmpCodeWithoutNaN(Condition);
1661 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1662 TBB, FBB, CurBB, TProb, FProb);
1663 SwitchCases.push_back(CB);
1668 // Create a CaseBlock record representing this branch.
1669 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1670 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1671 nullptr, TBB, FBB, CurBB, TProb, FProb);
1672 SwitchCases.push_back(CB);
1675 /// FindMergedConditions - If Cond is an expression like
1676 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1677 MachineBasicBlock *TBB,
1678 MachineBasicBlock *FBB,
1679 MachineBasicBlock *CurBB,
1680 MachineBasicBlock *SwitchBB,
1681 Instruction::BinaryOps Opc,
1682 BranchProbability TProb,
1683 BranchProbability FProb,
1685 // Skip over not part of the tree and remember to invert op and operands at
1687 if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1688 const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1689 if (InBlock(CondOp, CurBB->getBasicBlock())) {
1690 FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1696 const Instruction *BOp = dyn_cast<Instruction>(Cond);
1697 // Compute the effective opcode for Cond, taking into account whether it needs
1698 // to be inverted, e.g.
1699 // and (not (or A, B)), C
1701 // and (and (not A, not B), C)
1704 BOpc = BOp->getOpcode();
1706 if (BOpc == Instruction::And)
1707 BOpc = Instruction::Or;
1708 else if (BOpc == Instruction::Or)
1709 BOpc = Instruction::And;
1713 // If this node is not part of the or/and tree, emit it as a branch.
1714 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1715 BOpc != Opc || !BOp->hasOneUse() ||
1716 BOp->getParent() != CurBB->getBasicBlock() ||
1717 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1718 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1719 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1720 TProb, FProb, InvertCond);
1724 // Create TmpBB after CurBB.
1725 MachineFunction::iterator BBI(CurBB);
1726 MachineFunction &MF = DAG.getMachineFunction();
1727 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1728 CurBB->getParent()->insert(++BBI, TmpBB);
1730 if (Opc == Instruction::Or) {
1731 // Codegen X | Y as:
1740 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1741 // The requirement is that
1742 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1743 // = TrueProb for original BB.
1744 // Assuming the original probabilities are A and B, one choice is to set
1745 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1746 // A/(1+B) and 2B/(1+B). This choice assumes that
1747 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1748 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1749 // TmpBB, but the math is more complicated.
1751 auto NewTrueProb = TProb / 2;
1752 auto NewFalseProb = TProb / 2 + FProb;
1753 // Emit the LHS condition.
1754 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1755 NewTrueProb, NewFalseProb, InvertCond);
1757 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1758 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1759 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1760 // Emit the RHS condition into TmpBB.
1761 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1762 Probs[0], Probs[1], InvertCond);
1764 assert(Opc == Instruction::And && "Unknown merge op!");
1765 // Codegen X & Y as:
1773 // This requires creation of TmpBB after CurBB.
1775 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1776 // The requirement is that
1777 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1778 // = FalseProb for original BB.
1779 // Assuming the original probabilities are A and B, one choice is to set
1780 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1781 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1782 // TrueProb for BB1 * FalseProb for TmpBB.
1784 auto NewTrueProb = TProb + FProb / 2;
1785 auto NewFalseProb = FProb / 2;
1786 // Emit the LHS condition.
1787 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1788 NewTrueProb, NewFalseProb, InvertCond);
1790 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1791 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1792 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1793 // Emit the RHS condition into TmpBB.
1794 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1795 Probs[0], Probs[1], InvertCond);
1799 /// If the set of cases should be emitted as a series of branches, return true.
1800 /// If we should emit this as a bunch of and/or'd together conditions, return
1803 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1804 if (Cases.size() != 2) return true;
1806 // If this is two comparisons of the same values or'd or and'd together, they
1807 // will get folded into a single comparison, so don't emit two blocks.
1808 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1809 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1810 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1811 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1815 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1816 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1817 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1818 Cases[0].CC == Cases[1].CC &&
1819 isa<Constant>(Cases[0].CmpRHS) &&
1820 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1821 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1823 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1830 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1831 MachineBasicBlock *BrMBB = FuncInfo.MBB;
1833 // Update machine-CFG edges.
1834 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1836 if (I.isUnconditional()) {
1837 // Update machine-CFG edges.
1838 BrMBB->addSuccessor(Succ0MBB);
1840 // If this is not a fall-through branch or optimizations are switched off,
1842 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1843 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1844 MVT::Other, getControlRoot(),
1845 DAG.getBasicBlock(Succ0MBB)));
1850 // If this condition is one of the special cases we handle, do special stuff
1852 const Value *CondVal = I.getCondition();
1853 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1855 // If this is a series of conditions that are or'd or and'd together, emit
1856 // this as a sequence of branches instead of setcc's with and/or operations.
1857 // As long as jumps are not expensive, this should improve performance.
1858 // For example, instead of something like:
1871 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1872 Instruction::BinaryOps Opcode = BOp->getOpcode();
1873 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1874 !I.getMetadata(LLVMContext::MD_unpredictable) &&
1875 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1876 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1878 getEdgeProbability(BrMBB, Succ0MBB),
1879 getEdgeProbability(BrMBB, Succ1MBB),
1880 /*InvertCond=*/false);
1881 // If the compares in later blocks need to use values not currently
1882 // exported from this block, export them now. This block should always
1883 // be the first entry.
1884 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
1886 // Allow some cases to be rejected.
1887 if (ShouldEmitAsBranches(SwitchCases)) {
1888 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1889 ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
1890 ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
1893 // Emit the branch for this block.
1894 visitSwitchCase(SwitchCases[0], BrMBB);
1895 SwitchCases.erase(SwitchCases.begin());
1899 // Okay, we decided not to do this, remove any inserted MBB's and clear
1901 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1902 FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1904 SwitchCases.clear();
1908 // Create a CaseBlock record representing this branch.
1909 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1910 nullptr, Succ0MBB, Succ1MBB, BrMBB);
1912 // Use visitSwitchCase to actually insert the fast branch sequence for this
1914 visitSwitchCase(CB, BrMBB);
1917 /// visitSwitchCase - Emits the necessary code to represent a single node in
1918 /// the binary search tree resulting from lowering a switch instruction.
1919 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
1920 MachineBasicBlock *SwitchBB) {
1922 SDValue CondLHS = getValue(CB.CmpLHS);
1923 SDLoc dl = getCurSDLoc();
1925 // Build the setcc now.
1927 // Fold "(X == true)" to X and "(X == false)" to !X to
1928 // handle common cases produced by branch lowering.
1929 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1930 CB.CC == ISD::SETEQ)
1932 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1933 CB.CC == ISD::SETEQ) {
1934 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1935 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1937 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1939 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
1941 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1942 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1944 SDValue CmpOp = getValue(CB.CmpMHS);
1945 EVT VT = CmpOp.getValueType();
1947 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1948 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1951 SDValue SUB = DAG.getNode(ISD::SUB, dl,
1952 VT, CmpOp, DAG.getConstant(Low, dl, VT));
1953 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1954 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1958 // Update successor info
1959 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
1960 // TrueBB and FalseBB are always different unless the incoming IR is
1961 // degenerate. This only happens when running llc on weird IR.
1962 if (CB.TrueBB != CB.FalseBB)
1963 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
1964 SwitchBB->normalizeSuccProbs();
1966 // If the lhs block is the next block, invert the condition so that we can
1967 // fall through to the lhs instead of the rhs block.
1968 if (CB.TrueBB == NextBlock(SwitchBB)) {
1969 std::swap(CB.TrueBB, CB.FalseBB);
1970 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1971 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1974 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1975 MVT::Other, getControlRoot(), Cond,
1976 DAG.getBasicBlock(CB.TrueBB));
1978 // Insert the false branch. Do this even if it's a fall through branch,
1979 // this makes it easier to do DAG optimizations which require inverting
1980 // the branch condition.
1981 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1982 DAG.getBasicBlock(CB.FalseBB));
1984 DAG.setRoot(BrCond);
1987 /// visitJumpTable - Emit JumpTable node in the current MBB
1988 void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
1989 // Emit the code for the jump table
1990 assert(JT.Reg != -1U && "Should lower JT Header first!");
1991 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1992 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
1994 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1995 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1996 MVT::Other, Index.getValue(1),
1998 DAG.setRoot(BrJumpTable);
2001 /// visitJumpTableHeader - This function emits necessary code to produce index
2002 /// in the JumpTable from switch case.
2003 void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
2004 JumpTableHeader &JTH,
2005 MachineBasicBlock *SwitchBB) {
2006 SDLoc dl = getCurSDLoc();
2008 // Subtract the lowest switch case value from the value being switched on and
2009 // conditional branch to default mbb if the result is greater than the
2010 // difference between smallest and largest cases.
2011 SDValue SwitchOp = getValue(JTH.SValue);
2012 EVT VT = SwitchOp.getValueType();
2013 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2014 DAG.getConstant(JTH.First, dl, VT));
2016 // The SDNode we just created, which holds the value being switched on minus
2017 // the smallest case value, needs to be copied to a virtual register so it
2018 // can be used as an index into the jump table in a subsequent basic block.
2019 // This value may be smaller or larger than the target's pointer type, and
2020 // therefore require extension or truncating.
2021 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2022 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2024 unsigned JumpTableReg =
2025 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2026 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2027 JumpTableReg, SwitchOp);
2028 JT.Reg = JumpTableReg;
2030 // Emit the range check for the jump table, and branch to the default block
2031 // for the switch statement if the value being switched on exceeds the largest
2032 // case in the switch.
2033 SDValue CMP = DAG.getSetCC(
2034 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2035 Sub.getValueType()),
2036 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2038 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2039 MVT::Other, CopyTo, CMP,
2040 DAG.getBasicBlock(JT.Default));
2042 // Avoid emitting unnecessary branches to the next block.
2043 if (JT.MBB != NextBlock(SwitchBB))
2044 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2045 DAG.getBasicBlock(JT.MBB));
2047 DAG.setRoot(BrCond);
2050 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2051 /// variable if there exists one.
2052 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2054 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2055 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2056 MachineFunction &MF = DAG.getMachineFunction();
2057 Value *Global = TLI.getSDagStackGuard(*MF.getFunction()->getParent());
2058 MachineSDNode *Node =
2059 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2061 MachinePointerInfo MPInfo(Global);
2062 MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
2063 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2064 MachineMemOperand::MODereferenceable;
2065 *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
2066 DAG.getEVTAlignment(PtrTy));
2067 Node->setMemRefs(MemRefs, MemRefs + 1);
2069 return SDValue(Node, 0);
2072 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2073 /// tail spliced into a stack protector check success bb.
2075 /// For a high level explanation of how this fits into the stack protector
2076 /// generation see the comment on the declaration of class
2077 /// StackProtectorDescriptor.
2078 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2079 MachineBasicBlock *ParentBB) {
2081 // First create the loads to the guard/stack slot for the comparison.
2082 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2083 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2085 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2086 int FI = MFI.getStackProtectorIndex();
2089 SDLoc dl = getCurSDLoc();
2090 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2091 const Module &M = *ParentBB->getParent()->getFunction()->getParent();
2092 unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2094 // Generate code to load the content of the guard slot.
2095 SDValue StackSlot = DAG.getLoad(
2096 PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2097 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2098 MachineMemOperand::MOVolatile);
2100 // Retrieve guard check function, nullptr if instrumentation is inlined.
2101 if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2102 // The target provides a guard check function to validate the guard value.
2103 // Generate a call to that function with the content of the guard slot as
2105 auto *Fn = cast<Function>(GuardCheck);
2106 FunctionType *FnTy = Fn->getFunctionType();
2107 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2109 TargetLowering::ArgListTy Args;
2110 TargetLowering::ArgListEntry Entry;
2111 Entry.Node = StackSlot;
2112 Entry.Ty = FnTy->getParamType(0);
2113 if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2114 Entry.IsInReg = true;
2115 Args.push_back(Entry);
2117 TargetLowering::CallLoweringInfo CLI(DAG);
2118 CLI.setDebugLoc(getCurSDLoc())
2119 .setChain(DAG.getEntryNode())
2120 .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2121 getValue(GuardCheck), std::move(Args));
2123 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2124 DAG.setRoot(Result.second);
2128 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2129 // Otherwise, emit a volatile load to retrieve the stack guard value.
2130 SDValue Chain = DAG.getEntryNode();
2131 if (TLI.useLoadStackGuardNode()) {
2132 Guard = getLoadStackGuard(DAG, dl, Chain);
2134 const Value *IRGuard = TLI.getSDagStackGuard(M);
2135 SDValue GuardPtr = getValue(IRGuard);
2138 DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2139 Align, MachineMemOperand::MOVolatile);
2142 // Perform the comparison via a subtract/getsetcc.
2143 EVT VT = Guard.getValueType();
2144 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
2146 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2148 Sub.getValueType()),
2149 Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2151 // If the sub is not 0, then we know the guard/stackslot do not equal, so
2152 // branch to failure MBB.
2153 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2154 MVT::Other, StackSlot.getOperand(0),
2155 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2156 // Otherwise branch to success MBB.
2157 SDValue Br = DAG.getNode(ISD::BR, dl,
2159 DAG.getBasicBlock(SPD.getSuccessMBB()));
2164 /// Codegen the failure basic block for a stack protector check.
2166 /// A failure stack protector machine basic block consists simply of a call to
2167 /// __stack_chk_fail().
2169 /// For a high level explanation of how this fits into the stack protector
2170 /// generation see the comment on the declaration of class
2171 /// StackProtectorDescriptor.
2173 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2174 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2176 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2177 None, false, getCurSDLoc(), false, false).second;
2181 /// visitBitTestHeader - This function emits necessary code to produce value
2182 /// suitable for "bit tests"
2183 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2184 MachineBasicBlock *SwitchBB) {
2185 SDLoc dl = getCurSDLoc();
2187 // Subtract the minimum value
2188 SDValue SwitchOp = getValue(B.SValue);
2189 EVT VT = SwitchOp.getValueType();
2190 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2191 DAG.getConstant(B.First, dl, VT));
2194 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2195 SDValue RangeCmp = DAG.getSetCC(
2196 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2197 Sub.getValueType()),
2198 Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2200 // Determine the type of the test operands.
2201 bool UsePtrType = false;
2202 if (!TLI.isTypeLegal(VT))
2205 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2206 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2207 // Switch table case range are encoded into series of masks.
2208 // Just use pointer type, it's guaranteed to fit.
2214 VT = TLI.getPointerTy(DAG.getDataLayout());
2215 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2218 B.RegVT = VT.getSimpleVT();
2219 B.Reg = FuncInfo.CreateReg(B.RegVT);
2220 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2222 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2224 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2225 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2226 SwitchBB->normalizeSuccProbs();
2228 SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2229 MVT::Other, CopyTo, RangeCmp,
2230 DAG.getBasicBlock(B.Default));
2232 // Avoid emitting unnecessary branches to the next block.
2233 if (MBB != NextBlock(SwitchBB))
2234 BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2235 DAG.getBasicBlock(MBB));
2237 DAG.setRoot(BrRange);
2240 /// visitBitTestCase - this function produces one "bit test"
2241 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2242 MachineBasicBlock* NextMBB,
2243 BranchProbability BranchProbToNext,
2246 MachineBasicBlock *SwitchBB) {
2247 SDLoc dl = getCurSDLoc();
2249 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2251 unsigned PopCount = countPopulation(B.Mask);
2252 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2253 if (PopCount == 1) {
2254 // Testing for a single bit; just compare the shift count with what it
2255 // would need to be to shift a 1 bit in that position.
2257 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2258 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2260 } else if (PopCount == BB.Range) {
2261 // There is only one zero bit in the range, test for it directly.
2263 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2264 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2267 // Make desired shift
2268 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2269 DAG.getConstant(1, dl, VT), ShiftOp);
2271 // Emit bit tests and jumps
2272 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2273 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2275 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2276 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2279 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2280 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2281 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2282 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2283 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2284 // one as they are relative probabilities (and thus work more like weights),
2285 // and hence we need to normalize them to let the sum of them become one.
2286 SwitchBB->normalizeSuccProbs();
2288 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2289 MVT::Other, getControlRoot(),
2290 Cmp, DAG.getBasicBlock(B.TargetBB));
2292 // Avoid emitting unnecessary branches to the next block.
2293 if (NextMBB != NextBlock(SwitchBB))
2294 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2295 DAG.getBasicBlock(NextMBB));
2300 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2301 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2303 // Retrieve successors. Look through artificial IR level blocks like
2304 // catchswitch for successors.
2305 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2306 const BasicBlock *EHPadBB = I.getSuccessor(1);
2308 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2309 // have to do anything here to lower funclet bundles.
2310 assert(!I.hasOperandBundlesOtherThan(
2311 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2312 "Cannot lower invokes with arbitrary operand bundles yet!");
2314 const Value *Callee(I.getCalledValue());
2315 const Function *Fn = dyn_cast<Function>(Callee);
2316 if (isa<InlineAsm>(Callee))
2318 else if (Fn && Fn->isIntrinsic()) {
2319 switch (Fn->getIntrinsicID()) {
2321 llvm_unreachable("Cannot invoke this intrinsic");
2322 case Intrinsic::donothing:
2323 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2325 case Intrinsic::experimental_patchpoint_void:
2326 case Intrinsic::experimental_patchpoint_i64:
2327 visitPatchpoint(&I, EHPadBB);
2329 case Intrinsic::experimental_gc_statepoint:
2330 LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2333 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2334 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2335 // Eventually we will support lowering the @llvm.experimental.deoptimize
2336 // intrinsic, and right now there are no plans to support other intrinsics
2337 // with deopt state.
2338 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2340 LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2343 // If the value of the invoke is used outside of its defining block, make it
2344 // available as a virtual register.
2345 // We already took care of the exported value for the statepoint instruction
2346 // during call to the LowerStatepoint.
2347 if (!isStatepoint(I)) {
2348 CopyToExportRegsIfNeeded(&I);
2351 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2352 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2353 BranchProbability EHPadBBProb =
2354 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2355 : BranchProbability::getZero();
2356 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2358 // Update successor info.
2359 addSuccessorWithProb(InvokeMBB, Return);
2360 for (auto &UnwindDest : UnwindDests) {
2361 UnwindDest.first->setIsEHPad();
2362 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2364 InvokeMBB->normalizeSuccProbs();
2366 // Drop into normal successor.
2367 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2368 MVT::Other, getControlRoot(),
2369 DAG.getBasicBlock(Return)));
2372 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2373 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2376 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2377 assert(FuncInfo.MBB->isEHPad() &&
2378 "Call to landingpad not in landing pad!");
2380 MachineBasicBlock *MBB = FuncInfo.MBB;
2381 addLandingPadInfo(LP, *MBB);
2383 // If there aren't registers to copy the values into (e.g., during SjLj
2384 // exceptions), then don't bother to create these DAG nodes.
2385 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2386 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2387 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2388 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2391 // If landingpad's return type is token type, we don't create DAG nodes
2392 // for its exception pointer and selector value. The extraction of exception
2393 // pointer or selector value from token type landingpads is not currently
2395 if (LP.getType()->isTokenTy())
2398 SmallVector<EVT, 2> ValueVTs;
2399 SDLoc dl = getCurSDLoc();
2400 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2401 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2403 // Get the two live-in registers as SDValues. The physregs have already been
2404 // copied into virtual registers.
2406 if (FuncInfo.ExceptionPointerVirtReg) {
2407 Ops[0] = DAG.getZExtOrTrunc(
2408 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2409 FuncInfo.ExceptionPointerVirtReg,
2410 TLI.getPointerTy(DAG.getDataLayout())),
2413 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2415 Ops[1] = DAG.getZExtOrTrunc(
2416 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2417 FuncInfo.ExceptionSelectorVirtReg,
2418 TLI.getPointerTy(DAG.getDataLayout())),
2422 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2423 DAG.getVTList(ValueVTs), Ops);
2427 void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2429 for (const CaseCluster &CC : Clusters)
2430 assert(CC.Low == CC.High && "Input clusters must be single-case");
2433 std::sort(Clusters.begin(), Clusters.end(),
2434 [](const CaseCluster &a, const CaseCluster &b) {
2435 return a.Low->getValue().slt(b.Low->getValue());
2438 // Merge adjacent clusters with the same destination.
2439 const unsigned N = Clusters.size();
2440 unsigned DstIndex = 0;
2441 for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2442 CaseCluster &CC = Clusters[SrcIndex];
2443 const ConstantInt *CaseVal = CC.Low;
2444 MachineBasicBlock *Succ = CC.MBB;
2446 if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2447 (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2448 // If this case has the same successor and is a neighbour, merge it into
2449 // the previous cluster.
2450 Clusters[DstIndex - 1].High = CaseVal;
2451 Clusters[DstIndex - 1].Prob += CC.Prob;
2453 std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2454 sizeof(Clusters[SrcIndex]));
2457 Clusters.resize(DstIndex);
2460 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2461 MachineBasicBlock *Last) {
2463 for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2464 if (JTCases[i].first.HeaderBB == First)
2465 JTCases[i].first.HeaderBB = Last;
2467 // Update BitTestCases.
2468 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2469 if (BitTestCases[i].Parent == First)
2470 BitTestCases[i].Parent = Last;
2473 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2474 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2476 // Update machine-CFG edges with unique successors.
2477 SmallSet<BasicBlock*, 32> Done;
2478 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2479 BasicBlock *BB = I.getSuccessor(i);
2480 bool Inserted = Done.insert(BB).second;
2484 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2485 addSuccessorWithProb(IndirectBrMBB, Succ);
2487 IndirectBrMBB->normalizeSuccProbs();
2489 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2490 MVT::Other, getControlRoot(),
2491 getValue(I.getAddress())));
2494 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2495 if (DAG.getTarget().Options.TrapUnreachable)
2497 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2500 void SelectionDAGBuilder::visitFSub(const User &I) {
2501 // -0.0 - X --> fneg
2502 Type *Ty = I.getType();
2503 if (isa<Constant>(I.getOperand(0)) &&
2504 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2505 SDValue Op2 = getValue(I.getOperand(1));
2506 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2507 Op2.getValueType(), Op2));
2511 visitBinary(I, ISD::FSUB);
2514 /// Checks if the given instruction performs a vector reduction, in which case
2515 /// we have the freedom to alter the elements in the result as long as the
2516 /// reduction of them stays unchanged.
2517 static bool isVectorReductionOp(const User *I) {
2518 const Instruction *Inst = dyn_cast<Instruction>(I);
2519 if (!Inst || !Inst->getType()->isVectorTy())
2522 auto OpCode = Inst->getOpcode();
2524 case Instruction::Add:
2525 case Instruction::Mul:
2526 case Instruction::And:
2527 case Instruction::Or:
2528 case Instruction::Xor:
2530 case Instruction::FAdd:
2531 case Instruction::FMul:
2532 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2533 if (FPOp->getFastMathFlags().unsafeAlgebra())
2540 unsigned ElemNum = Inst->getType()->getVectorNumElements();
2541 unsigned ElemNumToReduce = ElemNum;
2543 // Do DFS search on the def-use chain from the given instruction. We only
2544 // allow four kinds of operations during the search until we reach the
2545 // instruction that extracts the first element from the vector:
2547 // 1. The reduction operation of the same opcode as the given instruction.
2551 // 3. ShuffleVector instruction together with a reduction operation that
2552 // does a partial reduction.
2554 // 4. ExtractElement that extracts the first element from the vector, and we
2555 // stop searching the def-use chain here.
2557 // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2558 // from 1-3 to the stack to continue the DFS. The given instruction is not
2559 // a reduction operation if we meet any other instructions other than those
2562 SmallVector<const User *, 16> UsersToVisit{Inst};
2563 SmallPtrSet<const User *, 16> Visited;
2564 bool ReduxExtracted = false;
2566 while (!UsersToVisit.empty()) {
2567 auto User = UsersToVisit.back();
2568 UsersToVisit.pop_back();
2569 if (!Visited.insert(User).second)
2572 for (const auto &U : User->users()) {
2573 auto Inst = dyn_cast<Instruction>(U);
2577 if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2578 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2579 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().unsafeAlgebra())
2581 UsersToVisit.push_back(U);
2582 } else if (const ShuffleVectorInst *ShufInst =
2583 dyn_cast<ShuffleVectorInst>(U)) {
2584 // Detect the following pattern: A ShuffleVector instruction together
2585 // with a reduction that do partial reduction on the first and second
2586 // ElemNumToReduce / 2 elements, and store the result in
2587 // ElemNumToReduce / 2 elements in another vector.
2589 unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2590 if (ResultElements < ElemNum)
2593 if (ElemNumToReduce == 1)
2595 if (!isa<UndefValue>(U->getOperand(1)))
2597 for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2598 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2600 for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2601 if (ShufInst->getMaskValue(i) != -1)
2604 // There is only one user of this ShuffleVector instruction, which
2605 // must be a reduction operation.
2606 if (!U->hasOneUse())
2609 auto U2 = dyn_cast<Instruction>(*U->user_begin());
2610 if (!U2 || U2->getOpcode() != OpCode)
2613 // Check operands of the reduction operation.
2614 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2615 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2616 UsersToVisit.push_back(U2);
2617 ElemNumToReduce /= 2;
2620 } else if (isa<ExtractElementInst>(U)) {
2621 // At this moment we should have reduced all elements in the vector.
2622 if (ElemNumToReduce != 1)
2625 const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2626 if (!Val || Val->getZExtValue() != 0)
2629 ReduxExtracted = true;
2634 return ReduxExtracted;
2637 void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2638 SDValue Op1 = getValue(I.getOperand(0));
2639 SDValue Op2 = getValue(I.getOperand(1));
2644 bool vec_redux = false;
2647 if (const OverflowingBinaryOperator *OFBinOp =
2648 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2649 nuw = OFBinOp->hasNoUnsignedWrap();
2650 nsw = OFBinOp->hasNoSignedWrap();
2652 if (const PossiblyExactOperator *ExactOp =
2653 dyn_cast<const PossiblyExactOperator>(&I))
2654 exact = ExactOp->isExact();
2655 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2656 FMF = FPOp->getFastMathFlags();
2658 if (isVectorReductionOp(&I)) {
2660 DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
2664 Flags.setExact(exact);
2665 Flags.setNoSignedWrap(nsw);
2666 Flags.setNoUnsignedWrap(nuw);
2667 Flags.setVectorReduction(vec_redux);
2668 Flags.setAllowReciprocal(FMF.allowReciprocal());
2669 Flags.setAllowContract(FMF.allowContract());
2670 Flags.setNoInfs(FMF.noInfs());
2671 Flags.setNoNaNs(FMF.noNaNs());
2672 Flags.setNoSignedZeros(FMF.noSignedZeros());
2673 Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2675 SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2677 setValue(&I, BinNodeValue);
2680 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2681 SDValue Op1 = getValue(I.getOperand(0));
2682 SDValue Op2 = getValue(I.getOperand(1));
2684 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2685 Op2.getValueType(), DAG.getDataLayout());
2687 // Coerce the shift amount to the right type if we can.
2688 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2689 unsigned ShiftSize = ShiftTy.getSizeInBits();
2690 unsigned Op2Size = Op2.getValueSizeInBits();
2691 SDLoc DL = getCurSDLoc();
2693 // If the operand is smaller than the shift count type, promote it.
2694 if (ShiftSize > Op2Size)
2695 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2697 // If the operand is larger than the shift count type but the shift
2698 // count type has enough bits to represent any shift value, truncate
2699 // it now. This is a common case and it exposes the truncate to
2700 // optimization early.
2701 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2702 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2703 // Otherwise we'll need to temporarily settle for some other convenient
2704 // type. Type legalization will make adjustments once the shiftee is split.
2706 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2713 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2715 if (const OverflowingBinaryOperator *OFBinOp =
2716 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2717 nuw = OFBinOp->hasNoUnsignedWrap();
2718 nsw = OFBinOp->hasNoSignedWrap();
2720 if (const PossiblyExactOperator *ExactOp =
2721 dyn_cast<const PossiblyExactOperator>(&I))
2722 exact = ExactOp->isExact();
2725 Flags.setExact(exact);
2726 Flags.setNoSignedWrap(nsw);
2727 Flags.setNoUnsignedWrap(nuw);
2728 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2733 void SelectionDAGBuilder::visitSDiv(const User &I) {
2734 SDValue Op1 = getValue(I.getOperand(0));
2735 SDValue Op2 = getValue(I.getOperand(1));
2738 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2739 cast<PossiblyExactOperator>(&I)->isExact());
2740 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2744 void SelectionDAGBuilder::visitICmp(const User &I) {
2745 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2746 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2747 predicate = IC->getPredicate();
2748 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2749 predicate = ICmpInst::Predicate(IC->getPredicate());
2750 SDValue Op1 = getValue(I.getOperand(0));
2751 SDValue Op2 = getValue(I.getOperand(1));
2752 ISD::CondCode Opcode = getICmpCondCode(predicate);
2754 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2756 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2759 void SelectionDAGBuilder::visitFCmp(const User &I) {
2760 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2761 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2762 predicate = FC->getPredicate();
2763 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2764 predicate = FCmpInst::Predicate(FC->getPredicate());
2765 SDValue Op1 = getValue(I.getOperand(0));
2766 SDValue Op2 = getValue(I.getOperand(1));
2767 ISD::CondCode Condition = getFCmpCondCode(predicate);
2769 // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
2770 // FIXME: We should propagate the fast-math-flags to the DAG node itself for
2771 // further optimization, but currently FMF is only applicable to binary nodes.
2772 if (TM.Options.NoNaNsFPMath)
2773 Condition = getFCmpCodeWithoutNaN(Condition);
2774 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2776 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2779 // Check if the condition of the select has one use or two users that are both
2780 // selects with the same condition.
2781 static bool hasOnlySelectUsers(const Value *Cond) {
2782 return all_of(Cond->users(), [](const Value *V) {
2783 return isa<SelectInst>(V);
2787 void SelectionDAGBuilder::visitSelect(const User &I) {
2788 SmallVector<EVT, 4> ValueVTs;
2789 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2791 unsigned NumValues = ValueVTs.size();
2792 if (NumValues == 0) return;
2794 SmallVector<SDValue, 4> Values(NumValues);
2795 SDValue Cond = getValue(I.getOperand(0));
2796 SDValue LHSVal = getValue(I.getOperand(1));
2797 SDValue RHSVal = getValue(I.getOperand(2));
2798 auto BaseOps = {Cond};
2799 ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2800 ISD::VSELECT : ISD::SELECT;
2802 // Min/max matching is only viable if all output VTs are the same.
2803 if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2804 EVT VT = ValueVTs[0];
2805 LLVMContext &Ctx = *DAG.getContext();
2806 auto &TLI = DAG.getTargetLoweringInfo();
2808 // We care about the legality of the operation after it has been type
2810 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2811 VT != TLI.getTypeToTransformTo(Ctx, VT))
2812 VT = TLI.getTypeToTransformTo(Ctx, VT);
2814 // If the vselect is legal, assume we want to leave this as a vector setcc +
2815 // vselect. Otherwise, if this is going to be scalarized, we want to see if
2816 // min/max is legal on the scalar type.
2817 bool UseScalarMinMax = VT.isVector() &&
2818 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2821 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2822 ISD::NodeType Opc = ISD::DELETED_NODE;
2823 switch (SPR.Flavor) {
2824 case SPF_UMAX: Opc = ISD::UMAX; break;
2825 case SPF_UMIN: Opc = ISD::UMIN; break;
2826 case SPF_SMAX: Opc = ISD::SMAX; break;
2827 case SPF_SMIN: Opc = ISD::SMIN; break;
2829 switch (SPR.NaNBehavior) {
2830 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2831 case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2832 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2833 case SPNB_RETURNS_ANY: {
2834 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2836 else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2838 else if (UseScalarMinMax)
2839 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2840 ISD::FMINNUM : ISD::FMINNAN;
2846 switch (SPR.NaNBehavior) {
2847 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
2848 case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2849 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2850 case SPNB_RETURNS_ANY:
2852 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2854 else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2856 else if (UseScalarMinMax)
2857 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2858 ISD::FMAXNUM : ISD::FMAXNAN;
2865 if (Opc != ISD::DELETED_NODE &&
2866 (TLI.isOperationLegalOrCustom(Opc, VT) ||
2868 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2869 // If the underlying comparison instruction is used by any other
2870 // instruction, the consumed instructions won't be destroyed, so it is
2871 // not profitable to convert to a min/max.
2872 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2874 LHSVal = getValue(LHS);
2875 RHSVal = getValue(RHS);
2880 for (unsigned i = 0; i != NumValues; ++i) {
2881 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2882 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2883 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2884 Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2885 LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2889 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
2890 DAG.getVTList(ValueVTs), Values));
2893 void SelectionDAGBuilder::visitTrunc(const User &I) {
2894 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2895 SDValue N = getValue(I.getOperand(0));
2896 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2898 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2901 void SelectionDAGBuilder::visitZExt(const User &I) {
2902 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2903 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2904 SDValue N = getValue(I.getOperand(0));
2905 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2907 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2910 void SelectionDAGBuilder::visitSExt(const User &I) {
2911 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2912 // SExt also can't be a cast to bool for same reason. So, nothing much to do
2913 SDValue N = getValue(I.getOperand(0));
2914 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2916 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2919 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2920 // FPTrunc is never a no-op cast, no need to check
2921 SDValue N = getValue(I.getOperand(0));
2922 SDLoc dl = getCurSDLoc();
2923 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2924 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2925 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2926 DAG.getTargetConstant(
2927 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2930 void SelectionDAGBuilder::visitFPExt(const User &I) {
2931 // FPExt is never a no-op cast, no need to check
2932 SDValue N = getValue(I.getOperand(0));
2933 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2935 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2938 void SelectionDAGBuilder::visitFPToUI(const User &I) {
2939 // FPToUI is never a no-op cast, no need to check
2940 SDValue N = getValue(I.getOperand(0));
2941 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2943 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2946 void SelectionDAGBuilder::visitFPToSI(const User &I) {
2947 // FPToSI is never a no-op cast, no need to check
2948 SDValue N = getValue(I.getOperand(0));
2949 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2951 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2954 void SelectionDAGBuilder::visitUIToFP(const User &I) {
2955 // UIToFP is never a no-op cast, no need to check
2956 SDValue N = getValue(I.getOperand(0));
2957 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2959 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2962 void SelectionDAGBuilder::visitSIToFP(const User &I) {
2963 // SIToFP is never a no-op cast, no need to check
2964 SDValue N = getValue(I.getOperand(0));
2965 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2967 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2970 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2971 // What to do depends on the size of the integer and the size of the pointer.
2972 // We can either truncate, zero extend, or no-op, accordingly.
2973 SDValue N = getValue(I.getOperand(0));
2974 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2976 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2979 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2980 // What to do depends on the size of the integer and the size of the pointer.
2981 // We can either truncate, zero extend, or no-op, accordingly.
2982 SDValue N = getValue(I.getOperand(0));
2983 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2985 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2988 void SelectionDAGBuilder::visitBitCast(const User &I) {
2989 SDValue N = getValue(I.getOperand(0));
2990 SDLoc dl = getCurSDLoc();
2991 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2994 // BitCast assures us that source and destination are the same size so this is
2995 // either a BITCAST or a no-op.
2996 if (DestVT != N.getValueType())
2997 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2998 DestVT, N)); // convert types.
2999 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3000 // might fold any kind of constant expression to an integer constant and that
3001 // is not what we are looking for. Only recognize a bitcast of a genuine
3002 // constant integer as an opaque constant.
3003 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3004 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3007 setValue(&I, N); // noop cast.
3010 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3011 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3012 const Value *SV = I.getOperand(0);
3013 SDValue N = getValue(SV);
3014 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3016 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3017 unsigned DestAS = I.getType()->getPointerAddressSpace();
3019 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3020 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3025 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3026 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3027 SDValue InVec = getValue(I.getOperand(0));
3028 SDValue InVal = getValue(I.getOperand(1));
3029 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3030 TLI.getVectorIdxTy(DAG.getDataLayout()));
3031 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3032 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3033 InVec, InVal, InIdx));
3036 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3037 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3038 SDValue InVec = getValue(I.getOperand(0));
3039 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3040 TLI.getVectorIdxTy(DAG.getDataLayout()));
3041 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3042 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3046 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3047 SDValue Src1 = getValue(I.getOperand(0));
3048 SDValue Src2 = getValue(I.getOperand(1));
3049 SDLoc DL = getCurSDLoc();
3051 SmallVector<int, 8> Mask;
3052 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3053 unsigned MaskNumElts = Mask.size();
3055 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3056 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3057 EVT SrcVT = Src1.getValueType();
3058 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3060 if (SrcNumElts == MaskNumElts) {
3061 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3065 // Normalize the shuffle vector since mask and vector length don't match.
3066 if (SrcNumElts < MaskNumElts) {
3067 // Mask is longer than the source vectors. We can use concatenate vector to
3068 // make the mask and vectors lengths match.
3070 if (MaskNumElts % SrcNumElts == 0) {
3071 // Mask length is a multiple of the source vector length.
3072 // Check if the shuffle is some kind of concatenation of the input
3074 unsigned NumConcat = MaskNumElts / SrcNumElts;
3075 bool IsConcat = true;
3076 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3077 for (unsigned i = 0; i != MaskNumElts; ++i) {
3081 // Ensure the indices in each SrcVT sized piece are sequential and that
3082 // the same source is used for the whole piece.
3083 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3084 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3085 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3089 // Remember which source this index came from.
3090 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3093 // The shuffle is concatenating multiple vectors together. Just emit
3094 // a CONCAT_VECTORS operation.
3096 SmallVector<SDValue, 8> ConcatOps;
3097 for (auto Src : ConcatSrcs) {
3099 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3101 ConcatOps.push_back(Src1);
3103 ConcatOps.push_back(Src2);
3105 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3110 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3111 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3112 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3115 // Pad both vectors with undefs to make them the same length as the mask.
3116 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3118 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3119 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3123 Src1 = Src1.isUndef()
3124 ? DAG.getUNDEF(PaddedVT)
3125 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3126 Src2 = Src2.isUndef()
3127 ? DAG.getUNDEF(PaddedVT)
3128 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3130 // Readjust mask for new input vector length.
3131 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3132 for (unsigned i = 0; i != MaskNumElts; ++i) {
3134 if (Idx >= (int)SrcNumElts)
3135 Idx -= SrcNumElts - PaddedMaskNumElts;
3139 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3141 // If the concatenated vector was padded, extract a subvector with the
3142 // correct number of elements.
3143 if (MaskNumElts != PaddedMaskNumElts)
3144 Result = DAG.getNode(
3145 ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3146 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3148 setValue(&I, Result);
3152 if (SrcNumElts > MaskNumElts) {
3153 // Analyze the access pattern of the vector to see if we can extract
3154 // two subvectors and do the shuffle.
3155 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3156 bool CanExtract = true;
3157 for (int Idx : Mask) {
3162 if (Idx >= (int)SrcNumElts) {
3167 // If all the indices come from the same MaskNumElts sized portion of
3168 // the sources we can use extract. Also make sure the extract wouldn't
3169 // extract past the end of the source.
3170 int NewStartIdx = alignDown(Idx, MaskNumElts);
3171 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3172 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3174 // Make sure we always update StartIdx as we use it to track if all
3175 // elements are undef.
3176 StartIdx[Input] = NewStartIdx;
3179 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3180 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3184 // Extract appropriate subvector and generate a vector shuffle
3185 for (unsigned Input = 0; Input < 2; ++Input) {
3186 SDValue &Src = Input == 0 ? Src1 : Src2;
3187 if (StartIdx[Input] < 0)
3188 Src = DAG.getUNDEF(VT);
3191 ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3192 DAG.getConstant(StartIdx[Input], DL,
3193 TLI.getVectorIdxTy(DAG.getDataLayout())));
3197 // Calculate new mask.
3198 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3199 for (int &Idx : MappedOps) {
3200 if (Idx >= (int)SrcNumElts)
3201 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3206 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3211 // We can't use either concat vectors or extract subvectors so fall back to
3212 // replacing the shuffle with extract and build vector.
3213 // to insert and build vector.
3214 EVT EltVT = VT.getVectorElementType();
3215 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3216 SmallVector<SDValue,8> Ops;
3217 for (int Idx : Mask) {
3221 Res = DAG.getUNDEF(EltVT);
3223 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3224 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3226 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3227 EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3233 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3236 void SelectionDAGBuilder::visitInsertValue(const User &I) {
3237 ArrayRef<unsigned> Indices;
3238 if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3239 Indices = IV->getIndices();
3241 Indices = cast<ConstantExpr>(&I)->getIndices();
3243 const Value *Op0 = I.getOperand(0);
3244 const Value *Op1 = I.getOperand(1);
3245 Type *AggTy = I.getType();
3246 Type *ValTy = Op1->getType();
3247 bool IntoUndef = isa<UndefValue>(Op0);
3248 bool FromUndef = isa<UndefValue>(Op1);
3250 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3252 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3253 SmallVector<EVT, 4> AggValueVTs;
3254 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3255 SmallVector<EVT, 4> ValValueVTs;
3256 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3258 unsigned NumAggValues = AggValueVTs.size();
3259 unsigned NumValValues = ValValueVTs.size();
3260 SmallVector<SDValue, 4> Values(NumAggValues);
3262 // Ignore an insertvalue that produces an empty object
3263 if (!NumAggValues) {
3264 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3268 SDValue Agg = getValue(Op0);
3270 // Copy the beginning value(s) from the original aggregate.
3271 for (; i != LinearIndex; ++i)
3272 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3273 SDValue(Agg.getNode(), Agg.getResNo() + i);
3274 // Copy values from the inserted value(s).
3276 SDValue Val = getValue(Op1);
3277 for (; i != LinearIndex + NumValValues; ++i)
3278 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3279 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3281 // Copy remaining value(s) from the original aggregate.
3282 for (; i != NumAggValues; ++i)
3283 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3284 SDValue(Agg.getNode(), Agg.getResNo() + i);
3286 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3287 DAG.getVTList(AggValueVTs), Values));
3290 void SelectionDAGBuilder::visitExtractValue(const User &I) {
3291 ArrayRef<unsigned> Indices;
3292 if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3293 Indices = EV->getIndices();
3295 Indices = cast<ConstantExpr>(&I)->getIndices();
3297 const Value *Op0 = I.getOperand(0);
3298 Type *AggTy = Op0->getType();
3299 Type *ValTy = I.getType();
3300 bool OutOfUndef = isa<UndefValue>(Op0);
3302 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3304 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3305 SmallVector<EVT, 4> ValValueVTs;
3306 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3308 unsigned NumValValues = ValValueVTs.size();
3310 // Ignore a extractvalue that produces an empty object
3311 if (!NumValValues) {
3312 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3316 SmallVector<SDValue, 4> Values(NumValValues);
3318 SDValue Agg = getValue(Op0);
3319 // Copy out the selected value(s).
3320 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3321 Values[i - LinearIndex] =
3323 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3324 SDValue(Agg.getNode(), Agg.getResNo() + i);
3326 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3327 DAG.getVTList(ValValueVTs), Values));
3330 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3331 Value *Op0 = I.getOperand(0);
3332 // Note that the pointer operand may be a vector of pointers. Take the scalar
3333 // element which holds a pointer.
3334 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3335 SDValue N = getValue(Op0);
3336 SDLoc dl = getCurSDLoc();
3338 // Normalize Vector GEP - all scalar operands should be converted to the
3340 unsigned VectorWidth = I.getType()->isVectorTy() ?
3341 cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3343 if (VectorWidth && !N.getValueType().isVector()) {
3344 LLVMContext &Context = *DAG.getContext();
3345 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3346 N = DAG.getSplatBuildVector(VT, dl, N);
3349 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3351 const Value *Idx = GTI.getOperand();
3352 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3353 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3356 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3358 // In an inbounds GEP with an offset that is nonnegative even when
3359 // interpreted as signed, assume there is no unsigned overflow.
3361 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3362 Flags.setNoUnsignedWrap(true);
3364 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3365 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3369 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
3370 unsigned PtrSize = PtrTy.getSizeInBits();
3371 APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3373 // If this is a scalar constant or a splat vector of constants,
3374 // handle it quickly.
3375 const auto *CI = dyn_cast<ConstantInt>(Idx);
3376 if (!CI && isa<ConstantDataVector>(Idx) &&
3377 cast<ConstantDataVector>(Idx)->getSplatValue())
3378 CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3383 APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
3384 LLVMContext &Context = *DAG.getContext();
3385 SDValue OffsVal = VectorWidth ?
3386 DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, PtrTy, VectorWidth)) :
3387 DAG.getConstant(Offs, dl, PtrTy);
3389 // In an inbouds GEP with an offset that is nonnegative even when
3390 // interpreted as signed, assume there is no unsigned overflow.
3392 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3393 Flags.setNoUnsignedWrap(true);
3395 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3399 // N = N + Idx * ElementSize;
3400 SDValue IdxN = getValue(Idx);
3402 if (!IdxN.getValueType().isVector() && VectorWidth) {
3403 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3404 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3407 // If the index is smaller or larger than intptr_t, truncate or extend
3409 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3411 // If this is a multiply by a power of two, turn it into a shl
3412 // immediately. This is a very common case.
3413 if (ElementSize != 1) {
3414 if (ElementSize.isPowerOf2()) {
3415 unsigned Amt = ElementSize.logBase2();
3416 IdxN = DAG.getNode(ISD::SHL, dl,
3417 N.getValueType(), IdxN,
3418 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3420 SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3421 IdxN = DAG.getNode(ISD::MUL, dl,
3422 N.getValueType(), IdxN, Scale);
3426 N = DAG.getNode(ISD::ADD, dl,
3427 N.getValueType(), N, IdxN);
3434 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3435 // If this is a fixed sized alloca in the entry block of the function,
3436 // allocate it statically on the stack.
3437 if (FuncInfo.StaticAllocaMap.count(&I))
3438 return; // getValue will auto-populate this.
3440 SDLoc dl = getCurSDLoc();
3441 Type *Ty = I.getAllocatedType();
3442 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3443 auto &DL = DAG.getDataLayout();
3444 uint64_t TySize = DL.getTypeAllocSize(Ty);
3446 std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3448 SDValue AllocSize = getValue(I.getArraySize());
3450 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
3451 if (AllocSize.getValueType() != IntPtr)
3452 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3454 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3456 DAG.getConstant(TySize, dl, IntPtr));
3458 // Handle alignment. If the requested alignment is less than or equal to
3459 // the stack alignment, ignore it. If the size is greater than or equal to
3460 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3461 unsigned StackAlign =
3462 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3463 if (Align <= StackAlign)
3466 // Round the size of the allocation up to the stack alignment size
3467 // by add SA-1 to the size. This doesn't overflow because we're computing
3468 // an address inside an alloca.
3470 Flags.setNoUnsignedWrap(true);
3471 AllocSize = DAG.getNode(ISD::ADD, dl,
3472 AllocSize.getValueType(), AllocSize,
3473 DAG.getIntPtrConstant(StackAlign - 1, dl), Flags);
3475 // Mask out the low bits for alignment purposes.
3476 AllocSize = DAG.getNode(ISD::AND, dl,
3477 AllocSize.getValueType(), AllocSize,
3478 DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
3481 SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
3482 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3483 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3485 DAG.setRoot(DSA.getValue(1));
3487 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3490 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3492 return visitAtomicLoad(I);
3494 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3495 const Value *SV = I.getOperand(0);
3496 if (TLI.supportSwiftError()) {
3497 // Swifterror values can come from either a function parameter with
3498 // swifterror attribute or an alloca with swifterror attribute.
3499 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3500 if (Arg->hasSwiftErrorAttr())
3501 return visitLoadFromSwiftError(I);
3504 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3505 if (Alloca->isSwiftError())
3506 return visitLoadFromSwiftError(I);
3510 SDValue Ptr = getValue(SV);
3512 Type *Ty = I.getType();
3514 bool isVolatile = I.isVolatile();
3515 bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3516 bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3517 bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3518 unsigned Alignment = I.getAlignment();
3521 I.getAAMetadata(AAInfo);
3522 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3524 SmallVector<EVT, 4> ValueVTs;
3525 SmallVector<uint64_t, 4> Offsets;
3526 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3527 unsigned NumValues = ValueVTs.size();
3532 bool ConstantMemory = false;
3533 if (isVolatile || NumValues > MaxParallelChains)
3534 // Serialize volatile loads with other side effects.
3536 else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3537 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3538 // Do not serialize (non-volatile) loads of constant memory with anything.
3539 Root = DAG.getEntryNode();
3540 ConstantMemory = true;
3542 // Do not serialize non-volatile loads against each other.
3543 Root = DAG.getRoot();
3546 SDLoc dl = getCurSDLoc();
3549 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3551 // An aggregate load cannot wrap around the address space, so offsets to its
3552 // parts don't wrap either.
3554 Flags.setNoUnsignedWrap(true);
3556 SmallVector<SDValue, 4> Values(NumValues);
3557 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3558 EVT PtrVT = Ptr.getValueType();
3559 unsigned ChainI = 0;
3560 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3561 // Serializing loads here may result in excessive register pressure, and
3562 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3563 // could recover a bit by hoisting nodes upward in the chain by recognizing
3564 // they are side-effect free or do not alias. The optimizer should really
3565 // avoid this case by converting large object/array copies to llvm.memcpy
3566 // (MaxParallelChains should always remain as failsafe).
3567 if (ChainI == MaxParallelChains) {
3568 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
3569 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3570 makeArrayRef(Chains.data(), ChainI));
3574 SDValue A = DAG.getNode(ISD::ADD, dl,
3576 DAG.getConstant(Offsets[i], dl, PtrVT),
3578 auto MMOFlags = MachineMemOperand::MONone;
3580 MMOFlags |= MachineMemOperand::MOVolatile;
3582 MMOFlags |= MachineMemOperand::MONonTemporal;
3584 MMOFlags |= MachineMemOperand::MOInvariant;
3585 if (isDereferenceable)
3586 MMOFlags |= MachineMemOperand::MODereferenceable;
3587 MMOFlags |= TLI.getMMOFlags(I);
3589 SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3590 MachinePointerInfo(SV, Offsets[i]), Alignment,
3591 MMOFlags, AAInfo, Ranges);
3594 Chains[ChainI] = L.getValue(1);
3597 if (!ConstantMemory) {
3598 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3599 makeArrayRef(Chains.data(), ChainI));
3603 PendingLoads.push_back(Chain);
3606 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3607 DAG.getVTList(ValueVTs), Values));
3610 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3611 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3612 "call visitStoreToSwiftError when backend supports swifterror");
3614 SmallVector<EVT, 4> ValueVTs;
3615 SmallVector<uint64_t, 4> Offsets;
3616 const Value *SrcV = I.getOperand(0);
3617 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3618 SrcV->getType(), ValueVTs, &Offsets);
3619 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3620 "expect a single EVT for swifterror");
3622 SDValue Src = getValue(SrcV);
3623 // Create a virtual register, then update the virtual register.
3624 unsigned VReg; bool CreatedVReg;
3625 std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3626 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3627 // Chain can be getRoot or getControlRoot.
3628 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3629 SDValue(Src.getNode(), Src.getResNo()));
3630 DAG.setRoot(CopyNode);
3632 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3635 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3636 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
3637 "call visitLoadFromSwiftError when backend supports swifterror");
3639 assert(!I.isVolatile() &&
3640 I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
3641 I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&
3642 "Support volatile, non temporal, invariant for load_from_swift_error");
3644 const Value *SV = I.getOperand(0);
3645 Type *Ty = I.getType();
3647 I.getAAMetadata(AAInfo);
3648 assert((!AA || !AA->pointsToConstantMemory(MemoryLocation(
3649 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&
3650 "load_from_swift_error should not be constant memory");
3652 SmallVector<EVT, 4> ValueVTs;
3653 SmallVector<uint64_t, 4> Offsets;
3654 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3655 ValueVTs, &Offsets);
3656 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
3657 "expect a single EVT for swifterror");
3659 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3660 SDValue L = DAG.getCopyFromReg(
3661 getRoot(), getCurSDLoc(),
3662 FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3668 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3670 return visitAtomicStore(I);
3672 const Value *SrcV = I.getOperand(0);
3673 const Value *PtrV = I.getOperand(1);
3675 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3676 if (TLI.supportSwiftError()) {
3677 // Swifterror values can come from either a function parameter with
3678 // swifterror attribute or an alloca with swifterror attribute.
3679 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3680 if (Arg->hasSwiftErrorAttr())
3681 return visitStoreToSwiftError(I);
3684 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3685 if (Alloca->isSwiftError())
3686 return visitStoreToSwiftError(I);
3690 SmallVector<EVT, 4> ValueVTs;
3691 SmallVector<uint64_t, 4> Offsets;
3692 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3693 SrcV->getType(), ValueVTs, &Offsets);
3694 unsigned NumValues = ValueVTs.size();
3698 // Get the lowered operands. Note that we do this after
3699 // checking if NumResults is zero, because with zero results
3700 // the operands won't have values in the map.
3701 SDValue Src = getValue(SrcV);
3702 SDValue Ptr = getValue(PtrV);
3704 SDValue Root = getRoot();
3705 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3706 SDLoc dl = getCurSDLoc();
3707 EVT PtrVT = Ptr.getValueType();
3708 unsigned Alignment = I.getAlignment();
3710 I.getAAMetadata(AAInfo);
3712 auto MMOFlags = MachineMemOperand::MONone;
3714 MMOFlags |= MachineMemOperand::MOVolatile;
3715 if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3716 MMOFlags |= MachineMemOperand::MONonTemporal;
3717 MMOFlags |= TLI.getMMOFlags(I);
3719 // An aggregate load cannot wrap around the address space, so offsets to its
3720 // parts don't wrap either.
3722 Flags.setNoUnsignedWrap(true);
3724 unsigned ChainI = 0;
3725 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3726 // See visitLoad comments.
3727 if (ChainI == MaxParallelChains) {
3728 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3729 makeArrayRef(Chains.data(), ChainI));
3733 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3734 DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3735 SDValue St = DAG.getStore(
3736 Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3737 MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3738 Chains[ChainI] = St;
3741 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3742 makeArrayRef(Chains.data(), ChainI));
3743 DAG.setRoot(StoreNode);
3746 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3747 bool IsCompressing) {
3748 SDLoc sdl = getCurSDLoc();
3750 auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3751 unsigned& Alignment) {
3752 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3753 Src0 = I.getArgOperand(0);
3754 Ptr = I.getArgOperand(1);
3755 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3756 Mask = I.getArgOperand(3);
3758 auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3759 unsigned& Alignment) {
3760 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3761 Src0 = I.getArgOperand(0);
3762 Ptr = I.getArgOperand(1);
3763 Mask = I.getArgOperand(2);
3767 Value *PtrOperand, *MaskOperand, *Src0Operand;
3770 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3772 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3774 SDValue Ptr = getValue(PtrOperand);
3775 SDValue Src0 = getValue(Src0Operand);
3776 SDValue Mask = getValue(MaskOperand);
3778 EVT VT = Src0.getValueType();
3780 Alignment = DAG.getEVTAlignment(VT);
3783 I.getAAMetadata(AAInfo);
3785 MachineMemOperand *MMO =
3786 DAG.getMachineFunction().
3787 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3788 MachineMemOperand::MOStore, VT.getStoreSize(),
3790 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3791 MMO, false /* Truncating */,
3793 DAG.setRoot(StoreNode);
3794 setValue(&I, StoreNode);
3797 // Get a uniform base for the Gather/Scatter intrinsic.
3798 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3799 // We try to represent it as a base pointer + vector of indices.
3800 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
3801 // The first operand of the GEP may be a single pointer or a vector of pointers
3803 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3805 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3806 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3808 // When the first GEP operand is a single pointer - it is the uniform base we
3809 // are looking for. If first operand of the GEP is a splat vector - we
3810 // extract the spalt value and use it as a uniform base.
3811 // In all other cases the function returns 'false'.
3813 static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3814 SelectionDAGBuilder* SDB) {
3816 SelectionDAG& DAG = SDB->DAG;
3817 LLVMContext &Context = *DAG.getContext();
3819 assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
3820 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3821 if (!GEP || GEP->getNumOperands() > 2)
3824 const Value *GEPPtr = GEP->getPointerOperand();
3825 if (!GEPPtr->getType()->isVectorTy())
3827 else if (!(Ptr = getSplatValue(GEPPtr)))
3830 Value *IndexVal = GEP->getOperand(1);
3832 // The operands of the GEP may be defined in another basic block.
3833 // In this case we'll not find nodes for the operands.
3834 if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3837 Base = SDB->getValue(Ptr);
3838 Index = SDB->getValue(IndexVal);
3840 // Suppress sign extension.
3841 if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3842 if (SDB->findValue(Sext->getOperand(0))) {
3843 IndexVal = Sext->getOperand(0);
3844 Index = SDB->getValue(IndexVal);
3847 if (!Index.getValueType().isVector()) {
3848 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3849 EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3850 Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3855 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3856 SDLoc sdl = getCurSDLoc();
3858 // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3859 const Value *Ptr = I.getArgOperand(1);
3860 SDValue Src0 = getValue(I.getArgOperand(0));
3861 SDValue Mask = getValue(I.getArgOperand(3));
3862 EVT VT = Src0.getValueType();
3863 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3865 Alignment = DAG.getEVTAlignment(VT);
3866 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3869 I.getAAMetadata(AAInfo);
3873 const Value *BasePtr = Ptr;
3874 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3876 const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3877 MachineMemOperand *MMO = DAG.getMachineFunction().
3878 getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3879 MachineMemOperand::MOStore, VT.getStoreSize(),
3882 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3883 Index = getValue(Ptr);
3885 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3886 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3888 DAG.setRoot(Scatter);
3889 setValue(&I, Scatter);
3892 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
3893 SDLoc sdl = getCurSDLoc();
3895 auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3896 unsigned& Alignment) {
3897 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3898 Ptr = I.getArgOperand(0);
3899 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
3900 Mask = I.getArgOperand(2);
3901 Src0 = I.getArgOperand(3);
3903 auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3904 unsigned& Alignment) {
3905 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
3906 Ptr = I.getArgOperand(0);
3908 Mask = I.getArgOperand(1);
3909 Src0 = I.getArgOperand(2);
3912 Value *PtrOperand, *MaskOperand, *Src0Operand;
3915 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3917 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3919 SDValue Ptr = getValue(PtrOperand);
3920 SDValue Src0 = getValue(Src0Operand);
3921 SDValue Mask = getValue(MaskOperand);
3923 EVT VT = Src0.getValueType();
3925 Alignment = DAG.getEVTAlignment(VT);
3928 I.getAAMetadata(AAInfo);
3929 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3931 // Do not serialize masked loads of constant memory with anything.
3932 bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
3933 PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
3934 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
3936 MachineMemOperand *MMO =
3937 DAG.getMachineFunction().
3938 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3939 MachineMemOperand::MOLoad, VT.getStoreSize(),
3940 Alignment, AAInfo, Ranges);
3942 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3943 ISD::NON_EXTLOAD, IsExpanding);
3945 SDValue OutChain = Load.getValue(1);
3946 DAG.setRoot(OutChain);
3951 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3952 SDLoc sdl = getCurSDLoc();
3954 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3955 const Value *Ptr = I.getArgOperand(0);
3956 SDValue Src0 = getValue(I.getArgOperand(3));
3957 SDValue Mask = getValue(I.getArgOperand(2));
3959 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3960 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3961 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3963 Alignment = DAG.getEVTAlignment(VT);
3966 I.getAAMetadata(AAInfo);
3967 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3969 SDValue Root = DAG.getRoot();
3972 const Value *BasePtr = Ptr;
3973 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3974 bool ConstantMemory = false;
3976 AA && AA->pointsToConstantMemory(MemoryLocation(
3977 BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
3979 // Do not serialize (non-volatile) loads of constant memory with anything.
3980 Root = DAG.getEntryNode();
3981 ConstantMemory = true;
3984 MachineMemOperand *MMO =
3985 DAG.getMachineFunction().
3986 getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3987 MachineMemOperand::MOLoad, VT.getStoreSize(),
3988 Alignment, AAInfo, Ranges);
3991 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3992 Index = getValue(Ptr);
3994 SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3995 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3998 SDValue OutChain = Gather.getValue(1);
3999 if (!ConstantMemory)
4000 PendingLoads.push_back(OutChain);
4001 setValue(&I, Gather);
4004 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4005 SDLoc dl = getCurSDLoc();
4006 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4007 AtomicOrdering FailureOrder = I.getFailureOrdering();
4008 SyncScope::ID SSID = I.getSyncScopeID();
4010 SDValue InChain = getRoot();
4012 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4013 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4014 SDValue L = DAG.getAtomicCmpSwap(
4015 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4016 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4017 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
4018 /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4020 SDValue OutChain = L.getValue(2);
4023 DAG.setRoot(OutChain);
4026 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4027 SDLoc dl = getCurSDLoc();
4029 switch (I.getOperation()) {
4030 default: llvm_unreachable("Unknown atomicrmw operation");
4031 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4032 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4033 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4034 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4035 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4036 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4037 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4038 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4039 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4040 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4041 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4043 AtomicOrdering Order = I.getOrdering();
4044 SyncScope::ID SSID = I.getSyncScopeID();
4046 SDValue InChain = getRoot();
4049 DAG.getAtomic(NT, dl,
4050 getValue(I.getValOperand()).getSimpleValueType(),
4052 getValue(I.getPointerOperand()),
4053 getValue(I.getValOperand()),
4054 I.getPointerOperand(),
4055 /* Alignment=*/ 0, Order, SSID);
4057 SDValue OutChain = L.getValue(1);
4060 DAG.setRoot(OutChain);
4063 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4064 SDLoc dl = getCurSDLoc();
4065 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4068 Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4069 TLI.getFenceOperandTy(DAG.getDataLayout()));
4070 Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4071 TLI.getFenceOperandTy(DAG.getDataLayout()));
4072 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4075 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4076 SDLoc dl = getCurSDLoc();
4077 AtomicOrdering Order = I.getOrdering();
4078 SyncScope::ID SSID = I.getSyncScopeID();
4080 SDValue InChain = getRoot();
4082 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4083 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4085 if (I.getAlignment() < VT.getSizeInBits() / 8)
4086 report_fatal_error("Cannot generate unaligned atomic load");
4088 MachineMemOperand *MMO =
4089 DAG.getMachineFunction().
4090 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4091 MachineMemOperand::MOVolatile |
4092 MachineMemOperand::MOLoad,
4094 I.getAlignment() ? I.getAlignment() :
4095 DAG.getEVTAlignment(VT),
4096 AAMDNodes(), nullptr, SSID, Order);
4098 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4100 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4101 getValue(I.getPointerOperand()), MMO);
4103 SDValue OutChain = L.getValue(1);
4106 DAG.setRoot(OutChain);
4109 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4110 SDLoc dl = getCurSDLoc();
4112 AtomicOrdering Order = I.getOrdering();
4113 SyncScope::ID SSID = I.getSyncScopeID();
4115 SDValue InChain = getRoot();
4117 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4119 TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4121 if (I.getAlignment() < VT.getSizeInBits() / 8)
4122 report_fatal_error("Cannot generate unaligned atomic store");
4125 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4127 getValue(I.getPointerOperand()),
4128 getValue(I.getValueOperand()),
4129 I.getPointerOperand(), I.getAlignment(),
4132 DAG.setRoot(OutChain);
4135 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4137 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4138 unsigned Intrinsic) {
4139 // Ignore the callsite's attributes. A specific call site may be marked with
4140 // readnone, but the lowering code will expect the chain based on the
4142 const Function *F = I.getCalledFunction();
4143 bool HasChain = !F->doesNotAccessMemory();
4144 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4146 // Build the operand list.
4147 SmallVector<SDValue, 8> Ops;
4148 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4150 // We don't need to serialize loads against other loads.
4151 Ops.push_back(DAG.getRoot());
4153 Ops.push_back(getRoot());
4157 // Info is set by getTgtMemInstrinsic
4158 TargetLowering::IntrinsicInfo Info;
4159 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4160 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
4162 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4163 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4164 Info.opc == ISD::INTRINSIC_W_CHAIN)
4165 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4166 TLI.getPointerTy(DAG.getDataLayout())));
4168 // Add all operands of the call to the operand list.
4169 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4170 SDValue Op = getValue(I.getArgOperand(i));
4174 SmallVector<EVT, 4> ValueVTs;
4175 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4178 ValueVTs.push_back(MVT::Other);
4180 SDVTList VTs = DAG.getVTList(ValueVTs);
4184 if (IsTgtIntrinsic) {
4185 // This is target intrinsic that touches memory
4186 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
4187 VTs, Ops, Info.memVT,
4188 MachinePointerInfo(Info.ptrVal, Info.offset),
4189 Info.align, Info.vol,
4190 Info.readMem, Info.writeMem, Info.size);
4191 } else if (!HasChain) {
4192 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4193 } else if (!I.getType()->isVoidTy()) {
4194 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4196 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4200 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4202 PendingLoads.push_back(Chain);
4207 if (!I.getType()->isVoidTy()) {
4208 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4209 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4210 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4212 Result = lowerRangeToAssertZExt(DAG, I, Result);
4214 setValue(&I, Result);
4218 /// GetSignificand - Get the significand and build it into a floating-point
4219 /// number with exponent of 1:
4221 /// Op = (Op & 0x007fffff) | 0x3f800000;
4223 /// where Op is the hexadecimal representation of floating point value.
4224 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4225 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4226 DAG.getConstant(0x007fffff, dl, MVT::i32));
4227 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4228 DAG.getConstant(0x3f800000, dl, MVT::i32));
4229 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4232 /// GetExponent - Get the exponent:
4234 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4236 /// where Op is the hexadecimal representation of floating point value.
4237 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4238 const TargetLowering &TLI, const SDLoc &dl) {
4239 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4240 DAG.getConstant(0x7f800000, dl, MVT::i32));
4241 SDValue t1 = DAG.getNode(
4242 ISD::SRL, dl, MVT::i32, t0,
4243 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4244 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4245 DAG.getConstant(127, dl, MVT::i32));
4246 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4249 /// getF32Constant - Get 32-bit floating point constant.
4250 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4252 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4256 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4257 SelectionDAG &DAG) {
4258 // TODO: What fast-math-flags should be set on the floating-point nodes?
4260 // IntegerPartOfX = ((int32_t)(t0);
4261 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4263 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4264 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4265 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4267 // IntegerPartOfX <<= 23;
4268 IntegerPartOfX = DAG.getNode(
4269 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4270 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4271 DAG.getDataLayout())));
4273 SDValue TwoToFractionalPartOfX;
4274 if (LimitFloatPrecision <= 6) {
4275 // For floating-point precision of 6:
4277 // TwoToFractionalPartOfX =
4279 // (0.735607626f + 0.252464424f * x) * x;
4281 // error 0.0144103317, which is 6 bits
4282 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4283 getF32Constant(DAG, 0x3e814304, dl));
4284 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4285 getF32Constant(DAG, 0x3f3c50c8, dl));
4286 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4287 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4288 getF32Constant(DAG, 0x3f7f5e7e, dl));
4289 } else if (LimitFloatPrecision <= 12) {
4290 // For floating-point precision of 12:
4292 // TwoToFractionalPartOfX =
4295 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4297 // error 0.000107046256, which is 13 to 14 bits
4298 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4299 getF32Constant(DAG, 0x3da235e3, dl));
4300 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4301 getF32Constant(DAG, 0x3e65b8f3, dl));
4302 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4303 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4304 getF32Constant(DAG, 0x3f324b07, dl));
4305 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4306 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4307 getF32Constant(DAG, 0x3f7ff8fd, dl));
4308 } else { // LimitFloatPrecision <= 18
4309 // For floating-point precision of 18:
4311 // TwoToFractionalPartOfX =
4315 // (0.554906021e-1f +
4316 // (0.961591928e-2f +
4317 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4318 // error 2.47208000*10^(-7), which is better than 18 bits
4319 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4320 getF32Constant(DAG, 0x3924b03e, dl));
4321 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4322 getF32Constant(DAG, 0x3ab24b87, dl));
4323 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4324 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4325 getF32Constant(DAG, 0x3c1d8c17, dl));
4326 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4327 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4328 getF32Constant(DAG, 0x3d634a1d, dl));
4329 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4330 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4331 getF32Constant(DAG, 0x3e75fe14, dl));
4332 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4333 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4334 getF32Constant(DAG, 0x3f317234, dl));
4335 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4336 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4337 getF32Constant(DAG, 0x3f800000, dl));
4340 // Add the exponent into the result in integer domain.
4341 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4342 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4343 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4346 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
4347 /// limited-precision mode.
4348 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4349 const TargetLowering &TLI) {
4350 if (Op.getValueType() == MVT::f32 &&
4351 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4353 // Put the exponent in the right bit position for later addition to the
4356 // #define LOG2OFe 1.4426950f
4357 // t0 = Op * LOG2OFe
4359 // TODO: What fast-math-flags should be set here?
4360 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4361 getF32Constant(DAG, 0x3fb8aa3b, dl));
4362 return getLimitedPrecisionExp2(t0, dl, DAG);
4365 // No special expansion.
4366 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4369 /// expandLog - Lower a log intrinsic. Handles the special sequences for
4370 /// limited-precision mode.
4371 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4372 const TargetLowering &TLI) {
4374 // TODO: What fast-math-flags should be set on the floating-point nodes?
4376 if (Op.getValueType() == MVT::f32 &&
4377 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4378 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4380 // Scale the exponent by log(2) [0.69314718f].
4381 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4382 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4383 getF32Constant(DAG, 0x3f317218, dl));
4385 // Get the significand and build it into a floating-point number with
4387 SDValue X = GetSignificand(DAG, Op1, dl);
4389 SDValue LogOfMantissa;
4390 if (LimitFloatPrecision <= 6) {
4391 // For floating-point precision of 6:
4395 // (1.4034025f - 0.23903021f * x) * x;
4397 // error 0.0034276066, which is better than 8 bits
4398 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4399 getF32Constant(DAG, 0xbe74c456, dl));
4400 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4401 getF32Constant(DAG, 0x3fb3a2b1, dl));
4402 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4403 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4404 getF32Constant(DAG, 0x3f949a29, dl));
4405 } else if (LimitFloatPrecision <= 12) {
4406 // For floating-point precision of 12:
4412 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4414 // error 0.000061011436, which is 14 bits
4415 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4416 getF32Constant(DAG, 0xbd67b6d6, dl));
4417 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4418 getF32Constant(DAG, 0x3ee4f4b8, dl));
4419 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4420 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4421 getF32Constant(DAG, 0x3fbc278b, dl));
4422 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4423 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4424 getF32Constant(DAG, 0x40348e95, dl));
4425 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4426 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4427 getF32Constant(DAG, 0x3fdef31a, dl));
4428 } else { // LimitFloatPrecision <= 18
4429 // For floating-point precision of 18:
4437 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4439 // error 0.0000023660568, which is better than 18 bits
4440 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4441 getF32Constant(DAG, 0xbc91e5ac, dl));
4442 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4443 getF32Constant(DAG, 0x3e4350aa, dl));
4444 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4445 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4446 getF32Constant(DAG, 0x3f60d3e3, dl));
4447 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4448 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4449 getF32Constant(DAG, 0x4011cdf0, dl));
4450 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4451 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4452 getF32Constant(DAG, 0x406cfd1c, dl));
4453 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4454 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4455 getF32Constant(DAG, 0x408797cb, dl));
4456 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4457 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4458 getF32Constant(DAG, 0x4006dcab, dl));
4461 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4464 // No special expansion.
4465 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4468 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4469 /// limited-precision mode.
4470 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4471 const TargetLowering &TLI) {
4473 // TODO: What fast-math-flags should be set on the floating-point nodes?
4475 if (Op.getValueType() == MVT::f32 &&
4476 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4477 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4479 // Get the exponent.
4480 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4482 // Get the significand and build it into a floating-point number with
4484 SDValue X = GetSignificand(DAG, Op1, dl);
4486 // Different possible minimax approximations of significand in
4487 // floating-point for various degrees of accuracy over [1,2].
4488 SDValue Log2ofMantissa;
4489 if (LimitFloatPrecision <= 6) {
4490 // For floating-point precision of 6:
4492 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4494 // error 0.0049451742, which is more than 7 bits
4495 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4496 getF32Constant(DAG, 0xbeb08fe0, dl));
4497 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4498 getF32Constant(DAG, 0x40019463, dl));
4499 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4500 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4501 getF32Constant(DAG, 0x3fd6633d, dl));
4502 } else if (LimitFloatPrecision <= 12) {
4503 // For floating-point precision of 12:
4509 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4511 // error 0.0000876136000, which is better than 13 bits
4512 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4513 getF32Constant(DAG, 0xbda7262e, dl));
4514 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4515 getF32Constant(DAG, 0x3f25280b, dl));
4516 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4517 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4518 getF32Constant(DAG, 0x4007b923, dl));
4519 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4520 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4521 getF32Constant(DAG, 0x40823e2f, dl));
4522 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4523 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4524 getF32Constant(DAG, 0x4020d29c, dl));
4525 } else { // LimitFloatPrecision <= 18
4526 // For floating-point precision of 18:
4535 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4537 // error 0.0000018516, which is better than 18 bits
4538 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4539 getF32Constant(DAG, 0xbcd2769e, dl));
4540 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4541 getF32Constant(DAG, 0x3e8ce0b9, dl));
4542 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4543 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4544 getF32Constant(DAG, 0x3fa22ae7, dl));
4545 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4546 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4547 getF32Constant(DAG, 0x40525723, dl));
4548 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4549 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4550 getF32Constant(DAG, 0x40aaf200, dl));
4551 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4552 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4553 getF32Constant(DAG, 0x40c39dad, dl));
4554 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4555 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4556 getF32Constant(DAG, 0x4042902c, dl));
4559 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4562 // No special expansion.
4563 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4566 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4567 /// limited-precision mode.
4568 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4569 const TargetLowering &TLI) {
4571 // TODO: What fast-math-flags should be set on the floating-point nodes?
4573 if (Op.getValueType() == MVT::f32 &&
4574 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4575 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4577 // Scale the exponent by log10(2) [0.30102999f].
4578 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4579 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4580 getF32Constant(DAG, 0x3e9a209a, dl));
4582 // Get the significand and build it into a floating-point number with
4584 SDValue X = GetSignificand(DAG, Op1, dl);
4586 SDValue Log10ofMantissa;
4587 if (LimitFloatPrecision <= 6) {
4588 // For floating-point precision of 6:
4590 // Log10ofMantissa =
4592 // (0.60948995f - 0.10380950f * x) * x;
4594 // error 0.0014886165, which is 6 bits
4595 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4596 getF32Constant(DAG, 0xbdd49a13, dl));
4597 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4598 getF32Constant(DAG, 0x3f1c0789, dl));
4599 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4600 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4601 getF32Constant(DAG, 0x3f011300, dl));
4602 } else if (LimitFloatPrecision <= 12) {
4603 // For floating-point precision of 12:
4605 // Log10ofMantissa =
4608 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4610 // error 0.00019228036, which is better than 12 bits
4611 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4612 getF32Constant(DAG, 0x3d431f31, dl));
4613 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4614 getF32Constant(DAG, 0x3ea21fb2, dl));
4615 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4616 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4617 getF32Constant(DAG, 0x3f6ae232, dl));
4618 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4619 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4620 getF32Constant(DAG, 0x3f25f7c3, dl));
4621 } else { // LimitFloatPrecision <= 18
4622 // For floating-point precision of 18:
4624 // Log10ofMantissa =
4629 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4631 // error 0.0000037995730, which is better than 18 bits
4632 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4633 getF32Constant(DAG, 0x3c5d51ce, dl));
4634 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4635 getF32Constant(DAG, 0x3e00685a, dl));
4636 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4637 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4638 getF32Constant(DAG, 0x3efb6798, dl));
4639 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4640 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4641 getF32Constant(DAG, 0x3f88d192, dl));
4642 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4643 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4644 getF32Constant(DAG, 0x3fc4316c, dl));
4645 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4646 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4647 getF32Constant(DAG, 0x3f57ce70, dl));
4650 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4653 // No special expansion.
4654 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4657 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4658 /// limited-precision mode.
4659 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4660 const TargetLowering &TLI) {
4661 if (Op.getValueType() == MVT::f32 &&
4662 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4663 return getLimitedPrecisionExp2(Op, dl, DAG);
4665 // No special expansion.
4666 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4669 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
4670 /// limited-precision mode with x == 10.0f.
4671 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4672 SelectionDAG &DAG, const TargetLowering &TLI) {
4673 bool IsExp10 = false;
4674 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4675 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4676 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4678 IsExp10 = LHSC->isExactlyValue(Ten);
4682 // TODO: What fast-math-flags should be set on the FMUL node?
4684 // Put the exponent in the right bit position for later addition to the
4687 // #define LOG2OF10 3.3219281f
4688 // t0 = Op * LOG2OF10;
4689 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4690 getF32Constant(DAG, 0x40549a78, dl));
4691 return getLimitedPrecisionExp2(t0, dl, DAG);
4694 // No special expansion.
4695 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4699 /// ExpandPowI - Expand a llvm.powi intrinsic.
4700 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4701 SelectionDAG &DAG) {
4702 // If RHS is a constant, we can expand this out to a multiplication tree,
4703 // otherwise we end up lowering to a call to __powidf2 (for example). When
4704 // optimizing for size, we only want to do this if the expansion would produce
4705 // a small number of multiplies, otherwise we do the full expansion.
4706 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4707 // Get the exponent as a positive value.
4708 unsigned Val = RHSC->getSExtValue();
4709 if ((int)Val < 0) Val = -Val;
4711 // powi(x, 0) -> 1.0
4713 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4715 const Function *F = DAG.getMachineFunction().getFunction();
4716 if (!F->optForSize() ||
4717 // If optimizing for size, don't insert too many multiplies.
4718 // This inserts up to 5 multiplies.
4719 countPopulation(Val) + Log2_32(Val) < 7) {
4720 // We use the simple binary decomposition method to generate the multiply
4721 // sequence. There are more optimal ways to do this (for example,
4722 // powi(x,15) generates one more multiply than it should), but this has
4723 // the benefit of being both really simple and much better than a libcall.
4724 SDValue Res; // Logically starts equal to 1.0
4725 SDValue CurSquare = LHS;
4726 // TODO: Intrinsics should have fast-math-flags that propagate to these
4731 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4733 Res = CurSquare; // 1.0*CurSquare.
4736 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4737 CurSquare, CurSquare);
4741 // If the original was negative, invert the result, producing 1/(x*x*x).
4742 if (RHSC->getSExtValue() < 0)
4743 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4744 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4749 // Otherwise, expand to a libcall.
4750 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4753 // getUnderlyingArgReg - Find underlying register used for a truncated or
4754 // bitcasted argument.
4755 static unsigned getUnderlyingArgReg(const SDValue &N) {
4756 switch (N.getOpcode()) {
4757 case ISD::CopyFromReg:
4758 return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4760 case ISD::AssertZext:
4761 case ISD::AssertSext:
4763 return getUnderlyingArgReg(N.getOperand(0));
4769 /// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4770 /// argument, create the corresponding DBG_VALUE machine instruction for it now.
4771 /// At the end of instruction selection, they will be inserted to the entry BB.
4772 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4773 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4774 DILocation *DL, int64_t Offset, bool IsDbgDeclare, const SDValue &N) {
4775 const Argument *Arg = dyn_cast<Argument>(V);
4779 MachineFunction &MF = DAG.getMachineFunction();
4780 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4782 // Ignore inlined function arguments here.
4784 // FIXME: Should we be checking DL->inlinedAt() to determine this?
4785 if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4788 bool IsIndirect = false;
4789 Optional<MachineOperand> Op;
4790 // Some arguments' frame index is recorded during argument lowering.
4791 int FI = FuncInfo.getArgumentFrameIndex(Arg);
4793 Op = MachineOperand::CreateFI(FI);
4795 if (!Op && N.getNode()) {
4796 unsigned Reg = getUnderlyingArgReg(N);
4797 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4798 MachineRegisterInfo &RegInfo = MF.getRegInfo();
4799 unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4804 Op = MachineOperand::CreateReg(Reg, false);
4805 IsIndirect = IsDbgDeclare;
4810 // Check if ValueMap has reg number.
4811 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4812 if (VMI != FuncInfo.ValueMap.end()) {
4813 Op = MachineOperand::CreateReg(VMI->second, false);
4814 IsIndirect = IsDbgDeclare;
4818 if (!Op && N.getNode())
4819 // Check if frame index is available.
4820 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4821 if (FrameIndexSDNode *FINode =
4822 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4823 Op = MachineOperand::CreateFI(FINode->getIndex());
4828 assert(Variable->isValidLocationForIntrinsic(DL) &&
4829 "Expected inlined-at fields to agree");
4831 FuncInfo.ArgDbgValues.push_back(
4832 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4833 Op->getReg(), Offset, Variable, Expr));
4835 FuncInfo.ArgDbgValues.push_back(
4836 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4839 .addMetadata(Variable)
4840 .addMetadata(Expr));
4845 /// Return the appropriate SDDbgValue based on N.
4846 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4847 DILocalVariable *Variable,
4848 DIExpression *Expr, int64_t Offset,
4850 unsigned DbgSDNodeOrder) {
4851 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
4852 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4853 // stack slot locations as such instead of as indirectly addressed
4855 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), 0, dl,
4858 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false,
4859 Offset, dl, DbgSDNodeOrder);
4862 // VisualStudio defines setjmp as _setjmp
4863 #if defined(_MSC_VER) && defined(setjmp) && \
4864 !defined(setjmp_undefined_for_msvc)
4865 # pragma push_macro("setjmp")
4867 # define setjmp_undefined_for_msvc
4870 /// Lower the call to the specified intrinsic function. If we want to emit this
4871 /// as a call to a named external function, return the name. Otherwise, lower it
4872 /// and return null.
4874 SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4875 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4876 SDLoc sdl = getCurSDLoc();
4877 DebugLoc dl = getCurDebugLoc();
4880 switch (Intrinsic) {
4882 // By default, turn this into a target intrinsic node.
4883 visitTargetIntrinsic(I, Intrinsic);
4885 case Intrinsic::vastart: visitVAStart(I); return nullptr;
4886 case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4887 case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4888 case Intrinsic::returnaddress:
4889 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
4890 TLI.getPointerTy(DAG.getDataLayout()),
4891 getValue(I.getArgOperand(0))));
4893 case Intrinsic::addressofreturnaddress:
4894 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
4895 TLI.getPointerTy(DAG.getDataLayout())));
4897 case Intrinsic::frameaddress:
4898 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
4899 TLI.getPointerTy(DAG.getDataLayout()),
4900 getValue(I.getArgOperand(0))));
4902 case Intrinsic::read_register: {
4903 Value *Reg = I.getArgOperand(0);
4904 SDValue Chain = getRoot();
4906 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4907 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4908 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4909 DAG.getVTList(VT, MVT::Other), Chain, RegName);
4911 DAG.setRoot(Res.getValue(1));
4914 case Intrinsic::write_register: {
4915 Value *Reg = I.getArgOperand(0);
4916 Value *RegValue = I.getArgOperand(1);
4917 SDValue Chain = getRoot();
4919 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4920 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
4921 RegName, getValue(RegValue)));
4924 case Intrinsic::setjmp:
4925 return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4926 case Intrinsic::longjmp:
4927 return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4928 case Intrinsic::memcpy: {
4929 SDValue Op1 = getValue(I.getArgOperand(0));
4930 SDValue Op2 = getValue(I.getArgOperand(1));
4931 SDValue Op3 = getValue(I.getArgOperand(2));
4932 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4934 Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4935 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4936 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4937 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4939 MachinePointerInfo(I.getArgOperand(0)),
4940 MachinePointerInfo(I.getArgOperand(1)));
4941 updateDAGForMaybeTailCall(MC);
4944 case Intrinsic::memset: {
4945 SDValue Op1 = getValue(I.getArgOperand(0));
4946 SDValue Op2 = getValue(I.getArgOperand(1));
4947 SDValue Op3 = getValue(I.getArgOperand(2));
4948 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4950 Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4951 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4952 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4953 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4954 isTC, MachinePointerInfo(I.getArgOperand(0)));
4955 updateDAGForMaybeTailCall(MS);
4958 case Intrinsic::memmove: {
4959 SDValue Op1 = getValue(I.getArgOperand(0));
4960 SDValue Op2 = getValue(I.getArgOperand(1));
4961 SDValue Op3 = getValue(I.getArgOperand(2));
4962 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4964 Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4965 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4966 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4967 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4968 isTC, MachinePointerInfo(I.getArgOperand(0)),
4969 MachinePointerInfo(I.getArgOperand(1)));
4970 updateDAGForMaybeTailCall(MM);
4973 case Intrinsic::memcpy_element_unordered_atomic: {
4974 const ElementUnorderedAtomicMemCpyInst &MI =
4975 cast<ElementUnorderedAtomicMemCpyInst>(I);
4976 SDValue Dst = getValue(MI.getRawDest());
4977 SDValue Src = getValue(MI.getRawSource());
4978 SDValue Length = getValue(MI.getLength());
4980 // Emit a library call.
4981 TargetLowering::ArgListTy Args;
4982 TargetLowering::ArgListEntry Entry;
4983 Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
4985 Args.push_back(Entry);
4988 Args.push_back(Entry);
4990 Entry.Ty = MI.getLength()->getType();
4991 Entry.Node = Length;
4992 Args.push_back(Entry);
4994 uint64_t ElementSizeConstant = MI.getElementSizeInBytes();
4995 RTLIB::Libcall LibraryCall =
4996 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSizeConstant);
4997 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
4998 report_fatal_error("Unsupported element size");
5000 TargetLowering::CallLoweringInfo CLI(DAG);
5001 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5002 TLI.getLibcallCallingConv(LibraryCall),
5003 Type::getVoidTy(*DAG.getContext()),
5004 DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall),
5005 TLI.getPointerTy(DAG.getDataLayout())),
5008 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
5009 DAG.setRoot(CallResult.second);
5012 case Intrinsic::memmove_element_unordered_atomic: {
5013 auto &MI = cast<ElementUnorderedAtomicMemMoveInst>(I);
5014 SDValue Dst = getValue(MI.getRawDest());
5015 SDValue Src = getValue(MI.getRawSource());
5016 SDValue Length = getValue(MI.getLength());
5018 // Emit a library call.
5019 TargetLowering::ArgListTy Args;
5020 TargetLowering::ArgListEntry Entry;
5021 Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
5023 Args.push_back(Entry);
5026 Args.push_back(Entry);
5028 Entry.Ty = MI.getLength()->getType();
5029 Entry.Node = Length;
5030 Args.push_back(Entry);
5032 uint64_t ElementSizeConstant = MI.getElementSizeInBytes();
5033 RTLIB::Libcall LibraryCall =
5034 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSizeConstant);
5035 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
5036 report_fatal_error("Unsupported element size");
5038 TargetLowering::CallLoweringInfo CLI(DAG);
5039 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5040 TLI.getLibcallCallingConv(LibraryCall),
5041 Type::getVoidTy(*DAG.getContext()),
5042 DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall),
5043 TLI.getPointerTy(DAG.getDataLayout())),
5046 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
5047 DAG.setRoot(CallResult.second);
5050 case Intrinsic::memset_element_unordered_atomic: {
5051 auto &MI = cast<ElementUnorderedAtomicMemSetInst>(I);
5052 SDValue Dst = getValue(MI.getRawDest());
5053 SDValue Val = getValue(MI.getValue());
5054 SDValue Length = getValue(MI.getLength());
5056 // Emit a library call.
5057 TargetLowering::ArgListTy Args;
5058 TargetLowering::ArgListEntry Entry;
5059 Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
5061 Args.push_back(Entry);
5063 Entry.Ty = Type::getInt8Ty(*DAG.getContext());
5065 Args.push_back(Entry);
5067 Entry.Ty = MI.getLength()->getType();
5068 Entry.Node = Length;
5069 Args.push_back(Entry);
5071 uint64_t ElementSizeConstant = MI.getElementSizeInBytes();
5072 RTLIB::Libcall LibraryCall =
5073 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSizeConstant);
5074 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
5075 report_fatal_error("Unsupported element size");
5077 TargetLowering::CallLoweringInfo CLI(DAG);
5078 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5079 TLI.getLibcallCallingConv(LibraryCall),
5080 Type::getVoidTy(*DAG.getContext()),
5081 DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall),
5082 TLI.getPointerTy(DAG.getDataLayout())),
5085 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
5086 DAG.setRoot(CallResult.second);
5089 case Intrinsic::dbg_declare: {
5090 const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
5091 DILocalVariable *Variable = DI.getVariable();
5092 DIExpression *Expression = DI.getExpression();
5093 const Value *Address = DI.getAddress();
5094 assert(Variable && "Missing variable");
5096 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5100 // Check if address has undef value.
5101 if (isa<UndefValue>(Address) ||
5102 (Address->use_empty() && !isa<Argument>(Address))) {
5103 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5107 // Byval arguments with frame indices were already handled after argument
5108 // lowering and before isel.
5110 dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
5111 if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
5114 SDValue &N = NodeMap[Address];
5115 if (!N.getNode() && isa<Argument>(Address))
5116 // Check unused arguments map.
5117 N = UnusedArgNodeMap[Address];
5120 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
5121 Address = BCI->getOperand(0);
5122 // Parameters are handled specially.
5123 bool isParameter = Variable->isParameter() || isa<Argument>(Address);
5124 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
5125 if (isParameter && FINode) {
5126 // Byval parameter. We have a frame index at this point.
5127 SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
5128 FINode->getIndex(), 0, dl, SDNodeOrder);
5129 } else if (isa<Argument>(Address)) {
5130 // Address is an argument, so try to emit its dbg value using
5131 // virtual register info from the FuncInfo.ValueMap.
5132 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, true, N);
5135 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
5136 true, 0, dl, SDNodeOrder);
5138 DAG.AddDbgValue(SDV, N.getNode(), isParameter);
5140 // If Address is an argument then try to emit its dbg value using
5141 // virtual register info from the FuncInfo.ValueMap.
5142 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, true,
5144 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
5149 case Intrinsic::dbg_value: {
5150 const DbgValueInst &DI = cast<DbgValueInst>(I);
5151 assert(DI.getVariable() && "Missing variable");
5153 DILocalVariable *Variable = DI.getVariable();
5154 DIExpression *Expression = DI.getExpression();
5155 uint64_t Offset = DI.getOffset();
5156 const Value *V = DI.getValue();
5161 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
5162 SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
5164 DAG.AddDbgValue(SDV, nullptr, false);
5168 // Do not use getValue() in here; we don't want to generate code at
5169 // this point if it hasn't been done yet.
5170 SDValue N = NodeMap[V];
5171 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
5172 N = UnusedArgNodeMap[V];
5174 if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset, false,
5177 SDV = getDbgValue(N, Variable, Expression, Offset, dl, SDNodeOrder);
5178 DAG.AddDbgValue(SDV, N.getNode(), false);
5182 if (!V->use_empty() ) {
5183 // Do not call getValue(V) yet, as we don't want to generate code.
5184 // Remember it for later.
5185 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
5186 DanglingDebugInfoMap[V] = DDI;
5190 DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n");
5191 DEBUG(dbgs() << " Last seen at:\n " << *V << "\n");
5195 case Intrinsic::eh_typeid_for: {
5196 // Find the type id for the given typeinfo.
5197 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5198 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5199 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5204 case Intrinsic::eh_return_i32:
5205 case Intrinsic::eh_return_i64:
5206 DAG.getMachineFunction().setCallsEHReturn(true);
5207 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5210 getValue(I.getArgOperand(0)),
5211 getValue(I.getArgOperand(1))));
5213 case Intrinsic::eh_unwind_init:
5214 DAG.getMachineFunction().setCallsUnwindInit(true);
5216 case Intrinsic::eh_dwarf_cfa: {
5217 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5218 TLI.getPointerTy(DAG.getDataLayout()),
5219 getValue(I.getArgOperand(0))));
5222 case Intrinsic::eh_sjlj_callsite: {
5223 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5224 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5225 assert(CI && "Non-constant call site value in eh.sjlj.callsite!");
5226 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
5228 MMI.setCurrentCallSite(CI->getZExtValue());
5231 case Intrinsic::eh_sjlj_functioncontext: {
5232 // Get and store the index of the function context.
5233 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5235 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5236 int FI = FuncInfo.StaticAllocaMap[FnCtx];
5237 MFI.setFunctionContextIndex(FI);
5240 case Intrinsic::eh_sjlj_setjmp: {
5243 Ops[1] = getValue(I.getArgOperand(0));
5244 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5245 DAG.getVTList(MVT::i32, MVT::Other), Ops);
5246 setValue(&I, Op.getValue(0));
5247 DAG.setRoot(Op.getValue(1));
5250 case Intrinsic::eh_sjlj_longjmp: {
5251 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5252 getRoot(), getValue(I.getArgOperand(0))));
5255 case Intrinsic::eh_sjlj_setup_dispatch: {
5256 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5261 case Intrinsic::masked_gather:
5262 visitMaskedGather(I);
5264 case Intrinsic::masked_load:
5267 case Intrinsic::masked_scatter:
5268 visitMaskedScatter(I);
5270 case Intrinsic::masked_store:
5271 visitMaskedStore(I);
5273 case Intrinsic::masked_expandload:
5274 visitMaskedLoad(I, true /* IsExpanding */);
5276 case Intrinsic::masked_compressstore:
5277 visitMaskedStore(I, true /* IsCompressing */);
5279 case Intrinsic::x86_mmx_pslli_w:
5280 case Intrinsic::x86_mmx_pslli_d:
5281 case Intrinsic::x86_mmx_pslli_q:
5282 case Intrinsic::x86_mmx_psrli_w:
5283 case Intrinsic::x86_mmx_psrli_d:
5284 case Intrinsic::x86_mmx_psrli_q:
5285 case Intrinsic::x86_mmx_psrai_w:
5286 case Intrinsic::x86_mmx_psrai_d: {
5287 SDValue ShAmt = getValue(I.getArgOperand(1));
5288 if (isa<ConstantSDNode>(ShAmt)) {
5289 visitTargetIntrinsic(I, Intrinsic);
5292 unsigned NewIntrinsic = 0;
5293 EVT ShAmtVT = MVT::v2i32;
5294 switch (Intrinsic) {
5295 case Intrinsic::x86_mmx_pslli_w:
5296 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5298 case Intrinsic::x86_mmx_pslli_d:
5299 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5301 case Intrinsic::x86_mmx_pslli_q:
5302 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5304 case Intrinsic::x86_mmx_psrli_w:
5305 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5307 case Intrinsic::x86_mmx_psrli_d:
5308 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5310 case Intrinsic::x86_mmx_psrli_q:
5311 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5313 case Intrinsic::x86_mmx_psrai_w:
5314 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5316 case Intrinsic::x86_mmx_psrai_d:
5317 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5319 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5322 // The vector shift intrinsics with scalars uses 32b shift amounts but
5323 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5325 // We must do this early because v2i32 is not a legal type.
5328 ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5329 ShAmt = DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5330 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5331 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5332 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5333 DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5334 getValue(I.getArgOperand(0)), ShAmt);
5338 case Intrinsic::powi:
5339 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5340 getValue(I.getArgOperand(1)), DAG));
5342 case Intrinsic::log:
5343 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5345 case Intrinsic::log2:
5346 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5348 case Intrinsic::log10:
5349 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5351 case Intrinsic::exp:
5352 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5354 case Intrinsic::exp2:
5355 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5357 case Intrinsic::pow:
5358 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5359 getValue(I.getArgOperand(1)), DAG, TLI));
5361 case Intrinsic::sqrt:
5362 case Intrinsic::fabs:
5363 case Intrinsic::sin:
5364 case Intrinsic::cos:
5365 case Intrinsic::floor:
5366 case Intrinsic::ceil:
5367 case Intrinsic::trunc:
5368 case Intrinsic::rint:
5369 case Intrinsic::nearbyint:
5370 case Intrinsic::round:
5371 case Intrinsic::canonicalize: {
5373 switch (Intrinsic) {
5374 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5375 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5376 case Intrinsic::fabs: Opcode = ISD::FABS; break;
5377 case Intrinsic::sin: Opcode = ISD::FSIN; break;
5378 case Intrinsic::cos: Opcode = ISD::FCOS; break;
5379 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5380 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5381 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5382 case Intrinsic::rint: Opcode = ISD::FRINT; break;
5383 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5384 case Intrinsic::round: Opcode = ISD::FROUND; break;
5385 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5388 setValue(&I, DAG.getNode(Opcode, sdl,
5389 getValue(I.getArgOperand(0)).getValueType(),
5390 getValue(I.getArgOperand(0))));
5393 case Intrinsic::minnum: {
5394 auto VT = getValue(I.getArgOperand(0)).getValueType();
5396 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT)
5399 setValue(&I, DAG.getNode(Opc, sdl, VT,
5400 getValue(I.getArgOperand(0)),
5401 getValue(I.getArgOperand(1))));
5404 case Intrinsic::maxnum: {
5405 auto VT = getValue(I.getArgOperand(0)).getValueType();
5407 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT)
5410 setValue(&I, DAG.getNode(Opc, sdl, VT,
5411 getValue(I.getArgOperand(0)),
5412 getValue(I.getArgOperand(1))));
5415 case Intrinsic::copysign:
5416 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5417 getValue(I.getArgOperand(0)).getValueType(),
5418 getValue(I.getArgOperand(0)),
5419 getValue(I.getArgOperand(1))));
5421 case Intrinsic::fma:
5422 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5423 getValue(I.getArgOperand(0)).getValueType(),
5424 getValue(I.getArgOperand(0)),
5425 getValue(I.getArgOperand(1)),
5426 getValue(I.getArgOperand(2))));
5428 case Intrinsic::experimental_constrained_fadd:
5429 case Intrinsic::experimental_constrained_fsub:
5430 case Intrinsic::experimental_constrained_fmul:
5431 case Intrinsic::experimental_constrained_fdiv:
5432 case Intrinsic::experimental_constrained_frem:
5433 case Intrinsic::experimental_constrained_sqrt:
5434 case Intrinsic::experimental_constrained_pow:
5435 case Intrinsic::experimental_constrained_powi:
5436 case Intrinsic::experimental_constrained_sin:
5437 case Intrinsic::experimental_constrained_cos:
5438 case Intrinsic::experimental_constrained_exp:
5439 case Intrinsic::experimental_constrained_exp2:
5440 case Intrinsic::experimental_constrained_log:
5441 case Intrinsic::experimental_constrained_log10:
5442 case Intrinsic::experimental_constrained_log2:
5443 case Intrinsic::experimental_constrained_rint:
5444 case Intrinsic::experimental_constrained_nearbyint:
5445 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
5447 case Intrinsic::fmuladd: {
5448 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5449 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5450 TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5451 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5452 getValue(I.getArgOperand(0)).getValueType(),
5453 getValue(I.getArgOperand(0)),
5454 getValue(I.getArgOperand(1)),
5455 getValue(I.getArgOperand(2))));
5457 // TODO: Intrinsic calls should have fast-math-flags.
5458 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5459 getValue(I.getArgOperand(0)).getValueType(),
5460 getValue(I.getArgOperand(0)),
5461 getValue(I.getArgOperand(1)));
5462 SDValue Add = DAG.getNode(ISD::FADD, sdl,
5463 getValue(I.getArgOperand(0)).getValueType(),
5465 getValue(I.getArgOperand(2)));
5470 case Intrinsic::convert_to_fp16:
5471 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5472 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5473 getValue(I.getArgOperand(0)),
5474 DAG.getTargetConstant(0, sdl,
5477 case Intrinsic::convert_from_fp16:
5478 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5479 TLI.getValueType(DAG.getDataLayout(), I.getType()),
5480 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5481 getValue(I.getArgOperand(0)))));
5483 case Intrinsic::pcmarker: {
5484 SDValue Tmp = getValue(I.getArgOperand(0));
5485 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5488 case Intrinsic::readcyclecounter: {
5489 SDValue Op = getRoot();
5490 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5491 DAG.getVTList(MVT::i64, MVT::Other), Op);
5493 DAG.setRoot(Res.getValue(1));
5496 case Intrinsic::bitreverse:
5497 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5498 getValue(I.getArgOperand(0)).getValueType(),
5499 getValue(I.getArgOperand(0))));
5501 case Intrinsic::bswap:
5502 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5503 getValue(I.getArgOperand(0)).getValueType(),
5504 getValue(I.getArgOperand(0))));
5506 case Intrinsic::cttz: {
5507 SDValue Arg = getValue(I.getArgOperand(0));
5508 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5509 EVT Ty = Arg.getValueType();
5510 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5514 case Intrinsic::ctlz: {
5515 SDValue Arg = getValue(I.getArgOperand(0));
5516 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5517 EVT Ty = Arg.getValueType();
5518 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5522 case Intrinsic::ctpop: {
5523 SDValue Arg = getValue(I.getArgOperand(0));
5524 EVT Ty = Arg.getValueType();
5525 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5528 case Intrinsic::stacksave: {
5529 SDValue Op = getRoot();
5531 ISD::STACKSAVE, sdl,
5532 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5534 DAG.setRoot(Res.getValue(1));
5537 case Intrinsic::stackrestore: {
5538 Res = getValue(I.getArgOperand(0));
5539 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5542 case Intrinsic::get_dynamic_area_offset: {
5543 SDValue Op = getRoot();
5544 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5545 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5546 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5549 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5551 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5557 case Intrinsic::stackguard: {
5558 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5559 MachineFunction &MF = DAG.getMachineFunction();
5560 const Module &M = *MF.getFunction()->getParent();
5561 SDValue Chain = getRoot();
5562 if (TLI.useLoadStackGuardNode()) {
5563 Res = getLoadStackGuard(DAG, sdl, Chain);
5565 const Value *Global = TLI.getSDagStackGuard(M);
5566 unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5567 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5568 MachinePointerInfo(Global, 0), Align,
5569 MachineMemOperand::MOVolatile);
5575 case Intrinsic::stackprotector: {
5576 // Emit code into the DAG to store the stack guard onto the stack.
5577 MachineFunction &MF = DAG.getMachineFunction();
5578 MachineFrameInfo &MFI = MF.getFrameInfo();
5579 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5580 SDValue Src, Chain = getRoot();
5582 if (TLI.useLoadStackGuardNode())
5583 Src = getLoadStackGuard(DAG, sdl, Chain);
5585 Src = getValue(I.getArgOperand(0)); // The guard's value.
5587 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5589 int FI = FuncInfo.StaticAllocaMap[Slot];
5590 MFI.setStackProtectorIndex(FI);
5592 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5594 // Store the stack protector onto the stack.
5595 Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5596 DAG.getMachineFunction(), FI),
5597 /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5602 case Intrinsic::objectsize: {
5603 // If we don't know by now, we're never going to know.
5604 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5606 assert(CI && "Non-constant type in __builtin_object_size?");
5608 SDValue Arg = getValue(I.getCalledValue());
5609 EVT Ty = Arg.getValueType();
5612 Res = DAG.getConstant(-1ULL, sdl, Ty);
5614 Res = DAG.getConstant(0, sdl, Ty);
5619 case Intrinsic::annotation:
5620 case Intrinsic::ptr_annotation:
5621 case Intrinsic::invariant_group_barrier:
5622 // Drop the intrinsic, but forward the value
5623 setValue(&I, getValue(I.getOperand(0)));
5625 case Intrinsic::assume:
5626 case Intrinsic::var_annotation:
5627 // Discard annotate attributes and assumptions
5630 case Intrinsic::init_trampoline: {
5631 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5635 Ops[1] = getValue(I.getArgOperand(0));
5636 Ops[2] = getValue(I.getArgOperand(1));
5637 Ops[3] = getValue(I.getArgOperand(2));
5638 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5639 Ops[5] = DAG.getSrcValue(F);
5641 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5646 case Intrinsic::adjust_trampoline: {
5647 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5648 TLI.getPointerTy(DAG.getDataLayout()),
5649 getValue(I.getArgOperand(0))));
5652 case Intrinsic::gcroot: {
5653 MachineFunction &MF = DAG.getMachineFunction();
5654 const Function *F = MF.getFunction();
5656 assert(F->hasGC() &&
5657 "only valid in functions with gc specified, enforced by Verifier");
5658 assert(GFI && "implied by previous");
5659 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5660 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5662 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5663 GFI->addStackRoot(FI->getIndex(), TypeMap);
5666 case Intrinsic::gcread:
5667 case Intrinsic::gcwrite:
5668 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
5669 case Intrinsic::flt_rounds:
5670 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5673 case Intrinsic::expect: {
5674 // Just replace __builtin_expect(exp, c) with EXP.
5675 setValue(&I, getValue(I.getArgOperand(0)));
5679 case Intrinsic::debugtrap:
5680 case Intrinsic::trap: {
5681 StringRef TrapFuncName =
5683 .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
5684 .getValueAsString();
5685 if (TrapFuncName.empty()) {
5686 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5687 ISD::TRAP : ISD::DEBUGTRAP;
5688 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5691 TargetLowering::ArgListTy Args;
5693 TargetLowering::CallLoweringInfo CLI(DAG);
5694 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5695 CallingConv::C, I.getType(),
5696 DAG.getExternalSymbol(TrapFuncName.data(),
5697 TLI.getPointerTy(DAG.getDataLayout())),
5700 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5701 DAG.setRoot(Result.second);
5705 case Intrinsic::uadd_with_overflow:
5706 case Intrinsic::sadd_with_overflow:
5707 case Intrinsic::usub_with_overflow:
5708 case Intrinsic::ssub_with_overflow:
5709 case Intrinsic::umul_with_overflow:
5710 case Intrinsic::smul_with_overflow: {
5712 switch (Intrinsic) {
5713 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5714 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5715 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5716 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5717 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5718 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5719 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5721 SDValue Op1 = getValue(I.getArgOperand(0));
5722 SDValue Op2 = getValue(I.getArgOperand(1));
5724 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5725 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5728 case Intrinsic::prefetch: {
5730 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5732 Ops[1] = getValue(I.getArgOperand(0));
5733 Ops[2] = getValue(I.getArgOperand(1));
5734 Ops[3] = getValue(I.getArgOperand(2));
5735 Ops[4] = getValue(I.getArgOperand(3));
5736 DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
5737 DAG.getVTList(MVT::Other), Ops,
5738 EVT::getIntegerVT(*Context, 8),
5739 MachinePointerInfo(I.getArgOperand(0)),
5741 false, /* volatile */
5743 rw==1)); /* write */
5746 case Intrinsic::lifetime_start:
5747 case Intrinsic::lifetime_end: {
5748 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
5749 // Stack coloring is not enabled in O0, discard region information.
5750 if (TM.getOptLevel() == CodeGenOpt::None)
5753 SmallVector<Value *, 4> Allocas;
5754 GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
5756 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
5757 E = Allocas.end(); Object != E; ++Object) {
5758 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
5760 // Could not find an Alloca.
5761 if (!LifetimeObject)
5764 // First check that the Alloca is static, otherwise it won't have a
5765 // valid frame index.
5766 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
5767 if (SI == FuncInfo.StaticAllocaMap.end())
5770 int FI = SI->second;
5775 DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true);
5776 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
5778 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
5783 case Intrinsic::invariant_start:
5784 // Discard region information.
5785 setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
5787 case Intrinsic::invariant_end:
5788 // Discard region information.
5790 case Intrinsic::clear_cache:
5791 return TLI.getClearCacheBuiltinName();
5792 case Intrinsic::donothing:
5795 case Intrinsic::experimental_stackmap: {
5799 case Intrinsic::experimental_patchpoint_void:
5800 case Intrinsic::experimental_patchpoint_i64: {
5801 visitPatchpoint(&I);
5804 case Intrinsic::experimental_gc_statepoint: {
5805 LowerStatepoint(ImmutableStatepoint(&I));
5808 case Intrinsic::experimental_gc_result: {
5809 visitGCResult(cast<GCResultInst>(I));
5812 case Intrinsic::experimental_gc_relocate: {
5813 visitGCRelocate(cast<GCRelocateInst>(I));
5816 case Intrinsic::instrprof_increment:
5817 llvm_unreachable("instrprof failed to lower an increment");
5818 case Intrinsic::instrprof_value_profile:
5819 llvm_unreachable("instrprof failed to lower a value profiling call");
5820 case Intrinsic::localescape: {
5821 MachineFunction &MF = DAG.getMachineFunction();
5822 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5824 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
5825 // is the same on all targets.
5826 for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
5827 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
5828 if (isa<ConstantPointerNull>(Arg))
5829 continue; // Skip null pointers. They represent a hole in index space.
5830 AllocaInst *Slot = cast<AllocaInst>(Arg);
5831 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
5832 "can only escape static allocas");
5833 int FI = FuncInfo.StaticAllocaMap[Slot];
5834 MCSymbol *FrameAllocSym =
5835 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5836 GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
5837 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
5838 TII->get(TargetOpcode::LOCAL_ESCAPE))
5839 .addSym(FrameAllocSym)
5846 case Intrinsic::localrecover: {
5847 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
5848 MachineFunction &MF = DAG.getMachineFunction();
5849 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
5851 // Get the symbol that defines the frame offset.
5852 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
5853 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
5854 unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX));
5855 MCSymbol *FrameAllocSym =
5856 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5857 GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
5859 // Create a MCSymbol for the label to avoid any target lowering
5860 // that would make this PC relative.
5861 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
5863 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
5865 // Add the offset to the FP.
5866 Value *FP = I.getArgOperand(1);
5867 SDValue FPVal = getValue(FP);
5868 SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
5874 case Intrinsic::eh_exceptionpointer:
5875 case Intrinsic::eh_exceptioncode: {
5876 // Get the exception pointer vreg, copy from it, and resize it to fit.
5877 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
5878 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5879 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
5880 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
5882 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
5883 if (Intrinsic == Intrinsic::eh_exceptioncode)
5884 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
5888 case Intrinsic::xray_customevent: {
5889 // Here we want to make sure that the intrinsic behaves as if it has a
5890 // specific calling convention, and only for x86_64.
5891 // FIXME: Support other platforms later.
5892 const auto &Triple = DAG.getTarget().getTargetTriple();
5893 if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
5896 SDLoc DL = getCurSDLoc();
5897 SmallVector<SDValue, 8> Ops;
5899 // We want to say that we always want the arguments in registers.
5900 SDValue LogEntryVal = getValue(I.getArgOperand(0));
5901 SDValue StrSizeVal = getValue(I.getArgOperand(1));
5902 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
5903 SDValue Chain = getRoot();
5904 Ops.push_back(LogEntryVal);
5905 Ops.push_back(StrSizeVal);
5906 Ops.push_back(Chain);
5908 // We need to enforce the calling convention for the callsite, so that
5909 // argument ordering is enforced correctly, and that register allocation can
5910 // see that some registers may be assumed clobbered and have to preserve
5911 // them across calls to the intrinsic.
5912 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
5914 SDValue patchableNode = SDValue(MN, 0);
5915 DAG.setRoot(patchableNode);
5916 setValue(&I, patchableNode);
5919 case Intrinsic::experimental_deoptimize:
5920 LowerDeoptimizeCall(&I);
5923 case Intrinsic::experimental_vector_reduce_fadd:
5924 case Intrinsic::experimental_vector_reduce_fmul:
5925 case Intrinsic::experimental_vector_reduce_add:
5926 case Intrinsic::experimental_vector_reduce_mul:
5927 case Intrinsic::experimental_vector_reduce_and:
5928 case Intrinsic::experimental_vector_reduce_or:
5929 case Intrinsic::experimental_vector_reduce_xor:
5930 case Intrinsic::experimental_vector_reduce_smax:
5931 case Intrinsic::experimental_vector_reduce_smin:
5932 case Intrinsic::experimental_vector_reduce_umax:
5933 case Intrinsic::experimental_vector_reduce_umin:
5934 case Intrinsic::experimental_vector_reduce_fmax:
5935 case Intrinsic::experimental_vector_reduce_fmin: {
5936 visitVectorReduce(I, Intrinsic);
5943 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
5944 const ConstrainedFPIntrinsic &FPI) {
5945 SDLoc sdl = getCurSDLoc();
5947 switch (FPI.getIntrinsicID()) {
5948 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
5949 case Intrinsic::experimental_constrained_fadd:
5950 Opcode = ISD::STRICT_FADD;
5952 case Intrinsic::experimental_constrained_fsub:
5953 Opcode = ISD::STRICT_FSUB;
5955 case Intrinsic::experimental_constrained_fmul:
5956 Opcode = ISD::STRICT_FMUL;
5958 case Intrinsic::experimental_constrained_fdiv:
5959 Opcode = ISD::STRICT_FDIV;
5961 case Intrinsic::experimental_constrained_frem:
5962 Opcode = ISD::STRICT_FREM;
5964 case Intrinsic::experimental_constrained_sqrt:
5965 Opcode = ISD::STRICT_FSQRT;
5967 case Intrinsic::experimental_constrained_pow:
5968 Opcode = ISD::STRICT_FPOW;
5970 case Intrinsic::experimental_constrained_powi:
5971 Opcode = ISD::STRICT_FPOWI;
5973 case Intrinsic::experimental_constrained_sin:
5974 Opcode = ISD::STRICT_FSIN;
5976 case Intrinsic::experimental_constrained_cos:
5977 Opcode = ISD::STRICT_FCOS;
5979 case Intrinsic::experimental_constrained_exp:
5980 Opcode = ISD::STRICT_FEXP;
5982 case Intrinsic::experimental_constrained_exp2:
5983 Opcode = ISD::STRICT_FEXP2;
5985 case Intrinsic::experimental_constrained_log:
5986 Opcode = ISD::STRICT_FLOG;
5988 case Intrinsic::experimental_constrained_log10:
5989 Opcode = ISD::STRICT_FLOG10;
5991 case Intrinsic::experimental_constrained_log2:
5992 Opcode = ISD::STRICT_FLOG2;
5994 case Intrinsic::experimental_constrained_rint:
5995 Opcode = ISD::STRICT_FRINT;
5997 case Intrinsic::experimental_constrained_nearbyint:
5998 Opcode = ISD::STRICT_FNEARBYINT;
6001 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6002 SDValue Chain = getRoot();
6003 SmallVector<EVT, 4> ValueVTs;
6004 ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
6005 ValueVTs.push_back(MVT::Other); // Out chain
6007 SDVTList VTs = DAG.getVTList(ValueVTs);
6009 if (FPI.isUnaryOp())
6010 Result = DAG.getNode(Opcode, sdl, VTs,
6011 { Chain, getValue(FPI.getArgOperand(0)) });
6013 Result = DAG.getNode(Opcode, sdl, VTs,
6014 { Chain, getValue(FPI.getArgOperand(0)),
6015 getValue(FPI.getArgOperand(1)) });
6017 assert(Result.getNode()->getNumValues() == 2);
6018 SDValue OutChain = Result.getValue(1);
6019 DAG.setRoot(OutChain);
6020 SDValue FPResult = Result.getValue(0);
6021 setValue(&FPI, FPResult);
6024 std::pair<SDValue, SDValue>
6025 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
6026 const BasicBlock *EHPadBB) {
6027 MachineFunction &MF = DAG.getMachineFunction();
6028 MachineModuleInfo &MMI = MF.getMMI();
6029 MCSymbol *BeginLabel = nullptr;
6032 // Insert a label before the invoke call to mark the try range. This can be
6033 // used to detect deletion of the invoke via the MachineModuleInfo.
6034 BeginLabel = MMI.getContext().createTempSymbol();
6036 // For SjLj, keep track of which landing pads go with which invokes
6037 // so as to maintain the ordering of pads in the LSDA.
6038 unsigned CallSiteIndex = MMI.getCurrentCallSite();
6039 if (CallSiteIndex) {
6040 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
6041 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
6043 // Now that the call site is handled, stop tracking it.
6044 MMI.setCurrentCallSite(0);
6047 // Both PendingLoads and PendingExports must be flushed here;
6048 // this call might not return.
6050 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
6052 CLI.setChain(getRoot());
6054 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6055 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6057 assert((CLI.IsTailCall || Result.second.getNode()) &&
6058 "Non-null chain expected with non-tail call!");
6059 assert((Result.second.getNode() || !Result.first.getNode()) &&
6060 "Null value expected with tail call!");
6062 if (!Result.second.getNode()) {
6063 // As a special case, a null chain means that a tail call has been emitted
6064 // and the DAG root is already updated.
6067 // Since there's no actual continuation from this block, nothing can be
6068 // relying on us setting vregs for them.
6069 PendingExports.clear();
6071 DAG.setRoot(Result.second);
6075 // Insert a label at the end of the invoke call to mark the try range. This
6076 // can be used to detect deletion of the invoke via the MachineModuleInfo.
6077 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
6078 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
6080 // Inform MachineModuleInfo of range.
6081 if (MF.hasEHFunclets()) {
6083 WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
6084 EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS->getInstruction()),
6085 BeginLabel, EndLabel);
6087 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
6094 void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
6096 const BasicBlock *EHPadBB) {
6097 auto &DL = DAG.getDataLayout();
6098 FunctionType *FTy = CS.getFunctionType();
6099 Type *RetTy = CS.getType();
6101 TargetLowering::ArgListTy Args;
6102 Args.reserve(CS.arg_size());
6104 const Value *SwiftErrorVal = nullptr;
6105 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6107 // We can't tail call inside a function with a swifterror argument. Lowering
6108 // does not support this yet. It would have to move into the swifterror
6109 // register before the call.
6110 auto *Caller = CS.getInstruction()->getParent()->getParent();
6111 if (TLI.supportSwiftError() &&
6112 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
6115 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
6117 TargetLowering::ArgListEntry Entry;
6118 const Value *V = *i;
6121 if (V->getType()->isEmptyTy())
6124 SDValue ArgNode = getValue(V);
6125 Entry.Node = ArgNode; Entry.Ty = V->getType();
6127 Entry.setAttributes(&CS, i - CS.arg_begin());
6129 // Use swifterror virtual register as input to the call.
6130 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
6132 // We find the virtual register for the actual swifterror argument.
6133 // Instead of using the Value, we use the virtual register instead.
6134 Entry.Node = DAG.getRegister(FuncInfo
6135 .getOrCreateSwiftErrorVRegUseAt(
6136 CS.getInstruction(), FuncInfo.MBB, V)
6138 EVT(TLI.getPointerTy(DL)));
6141 Args.push_back(Entry);
6143 // If we have an explicit sret argument that is an Instruction, (i.e., it
6144 // might point to function-local memory), we can't meaningfully tail-call.
6145 if (Entry.IsSRet && isa<Instruction>(V))
6149 // Check if target-independent constraints permit a tail call here.
6150 // Target-dependent constraints are checked within TLI->LowerCallTo.
6151 if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
6154 // Disable tail calls if there is an swifterror argument. Targets have not
6155 // been updated to support tail calls.
6156 if (TLI.supportSwiftError() && SwiftErrorVal)
6159 TargetLowering::CallLoweringInfo CLI(DAG);
6160 CLI.setDebugLoc(getCurSDLoc())
6161 .setChain(getRoot())
6162 .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
6163 .setTailCall(isTailCall)
6164 .setConvergent(CS.isConvergent());
6165 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
6167 if (Result.first.getNode()) {
6168 const Instruction *Inst = CS.getInstruction();
6169 Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
6170 setValue(Inst, Result.first);
6173 // The last element of CLI.InVals has the SDValue for swifterror return.
6174 // Here we copy it to a virtual register and update SwiftErrorMap for
6176 if (SwiftErrorVal && TLI.supportSwiftError()) {
6177 // Get the last element of InVals.
6178 SDValue Src = CLI.InVals.back();
6179 unsigned VReg; bool CreatedVReg;
6180 std::tie(VReg, CreatedVReg) =
6181 FuncInfo.getOrCreateSwiftErrorVRegDefAt(CS.getInstruction());
6182 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
6183 // We update the virtual register for the actual swifterror argument.
6185 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
6186 DAG.setRoot(CopyNode);
6190 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
6191 SelectionDAGBuilder &Builder) {
6193 // Check to see if this load can be trivially constant folded, e.g. if the
6194 // input is from a string literal.
6195 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
6196 // Cast pointer to the type we really want to load.
6198 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
6199 if (LoadVT.isVector())
6200 LoadTy = VectorType::get(LoadTy, LoadVT.getVectorNumElements());
6202 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
6203 PointerType::getUnqual(LoadTy));
6205 if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
6206 const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
6207 return Builder.getValue(LoadCst);
6210 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
6211 // still constant memory, the input chain can be the entry node.
6213 bool ConstantMemory = false;
6215 // Do not serialize (non-volatile) loads of constant memory with anything.
6216 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
6217 Root = Builder.DAG.getEntryNode();
6218 ConstantMemory = true;
6220 // Do not serialize non-volatile loads against each other.
6221 Root = Builder.DAG.getRoot();
6224 SDValue Ptr = Builder.getValue(PtrVal);
6225 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
6226 Ptr, MachinePointerInfo(PtrVal),
6227 /* Alignment = */ 1);
6229 if (!ConstantMemory)
6230 Builder.PendingLoads.push_back(LoadVal.getValue(1));
6234 /// Record the value for an instruction that produces an integer result,
6235 /// converting the type where necessary.
6236 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
6239 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6242 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
6244 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
6245 setValue(&I, Value);
6248 /// See if we can lower a memcmp call into an optimized form. If so, return
6249 /// true and lower it. Otherwise return false, and it will be lowered like a
6251 /// The caller already checked that \p I calls the appropriate LibFunc with a
6252 /// correct prototype.
6253 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6254 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6255 const Value *Size = I.getArgOperand(2);
6256 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6257 if (CSize && CSize->getZExtValue() == 0) {
6258 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6260 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6264 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6265 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
6266 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
6267 getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
6268 if (Res.first.getNode()) {
6269 processIntegerCallValue(I, Res.first, true);
6270 PendingLoads.push_back(Res.second);
6274 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
6275 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
6276 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
6279 // If the target has a fast compare for the given size, it will return a
6280 // preferred load type for that size. Require that the load VT is legal and
6281 // that the target supports unaligned loads of that type. Otherwise, return
6283 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
6284 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6285 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
6286 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
6287 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6288 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6289 // TODO: Check alignment of src and dest ptrs.
6290 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6291 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6292 if (!TLI.isTypeLegal(LVT) ||
6293 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
6294 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
6295 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
6301 // This turns into unaligned loads. We only do this if the target natively
6302 // supports the MVT we'll be loading or if it is small enough (<= 4) that
6303 // we'll only produce a small number of byte loads.
6305 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
6306 switch (NumBitsToCompare) {
6318 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
6322 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
6325 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
6326 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
6328 // Bitcast to a wide integer type if the loads are vectors.
6329 if (LoadVT.isVector()) {
6330 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
6331 LoadL = DAG.getBitcast(CmpVT, LoadL);
6332 LoadR = DAG.getBitcast(CmpVT, LoadR);
6335 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
6336 processIntegerCallValue(I, Cmp, false);
6340 /// See if we can lower a memchr call into an optimized form. If so, return
6341 /// true and lower it. Otherwise return false, and it will be lowered like a
6343 /// The caller already checked that \p I calls the appropriate LibFunc with a
6344 /// correct prototype.
6345 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6346 const Value *Src = I.getArgOperand(0);
6347 const Value *Char = I.getArgOperand(1);
6348 const Value *Length = I.getArgOperand(2);
6350 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6351 std::pair<SDValue, SDValue> Res =
6352 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6353 getValue(Src), getValue(Char), getValue(Length),
6354 MachinePointerInfo(Src));
6355 if (Res.first.getNode()) {
6356 setValue(&I, Res.first);
6357 PendingLoads.push_back(Res.second);
6364 /// See if we can lower a mempcpy call into an optimized form. If so, return
6365 /// true and lower it. Otherwise return false, and it will be lowered like a
6367 /// The caller already checked that \p I calls the appropriate LibFunc with a
6368 /// correct prototype.
6369 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6370 SDValue Dst = getValue(I.getArgOperand(0));
6371 SDValue Src = getValue(I.getArgOperand(1));
6372 SDValue Size = getValue(I.getArgOperand(2));
6374 unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6375 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6376 unsigned Align = std::min(DstAlign, SrcAlign);
6377 if (Align == 0) // Alignment of one or both could not be inferred.
6378 Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6381 SDLoc sdl = getCurSDLoc();
6383 // In the mempcpy context we need to pass in a false value for isTailCall
6384 // because the return pointer needs to be adjusted by the size of
6385 // the copied memory.
6386 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6387 false, /*isTailCall=*/false,
6388 MachinePointerInfo(I.getArgOperand(0)),
6389 MachinePointerInfo(I.getArgOperand(1)));
6390 assert(MC.getNode() != nullptr &&
6391 "** memcpy should not be lowered as TailCall in mempcpy context **");
6394 // Check if Size needs to be truncated or extended.
6395 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6397 // Adjust return pointer to point just past the last dst byte.
6398 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6400 setValue(&I, DstPlusSize);
6404 /// See if we can lower a strcpy call into an optimized form. If so, return
6405 /// true and lower it, otherwise return false and it will be lowered like a
6407 /// The caller already checked that \p I calls the appropriate LibFunc with a
6408 /// correct prototype.
6409 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6410 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6412 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6413 std::pair<SDValue, SDValue> Res =
6414 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6415 getValue(Arg0), getValue(Arg1),
6416 MachinePointerInfo(Arg0),
6417 MachinePointerInfo(Arg1), isStpcpy);
6418 if (Res.first.getNode()) {
6419 setValue(&I, Res.first);
6420 DAG.setRoot(Res.second);
6427 /// See if we can lower a strcmp call into an optimized form. If so, return
6428 /// true and lower it, otherwise return false and it will be lowered like a
6430 /// The caller already checked that \p I calls the appropriate LibFunc with a
6431 /// correct prototype.
6432 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6433 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6435 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6436 std::pair<SDValue, SDValue> Res =
6437 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6438 getValue(Arg0), getValue(Arg1),
6439 MachinePointerInfo(Arg0),
6440 MachinePointerInfo(Arg1));
6441 if (Res.first.getNode()) {
6442 processIntegerCallValue(I, Res.first, true);
6443 PendingLoads.push_back(Res.second);
6450 /// See if we can lower a strlen call into an optimized form. If so, return
6451 /// true and lower it, otherwise return false and it will be lowered like a
6453 /// The caller already checked that \p I calls the appropriate LibFunc with a
6454 /// correct prototype.
6455 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6456 const Value *Arg0 = I.getArgOperand(0);
6458 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6459 std::pair<SDValue, SDValue> Res =
6460 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6461 getValue(Arg0), MachinePointerInfo(Arg0));
6462 if (Res.first.getNode()) {
6463 processIntegerCallValue(I, Res.first, false);
6464 PendingLoads.push_back(Res.second);
6471 /// See if we can lower a strnlen call into an optimized form. If so, return
6472 /// true and lower it, otherwise return false and it will be lowered like a
6474 /// The caller already checked that \p I calls the appropriate LibFunc with a
6475 /// correct prototype.
6476 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6477 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6479 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6480 std::pair<SDValue, SDValue> Res =
6481 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6482 getValue(Arg0), getValue(Arg1),
6483 MachinePointerInfo(Arg0));
6484 if (Res.first.getNode()) {
6485 processIntegerCallValue(I, Res.first, false);
6486 PendingLoads.push_back(Res.second);
6493 /// See if we can lower a unary floating-point operation into an SDNode with
6494 /// the specified Opcode. If so, return true and lower it, otherwise return
6495 /// false and it will be lowered like a normal call.
6496 /// The caller already checked that \p I calls the appropriate LibFunc with a
6497 /// correct prototype.
6498 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6500 // We already checked this call's prototype; verify it doesn't modify errno.
6501 if (!I.onlyReadsMemory())
6504 SDValue Tmp = getValue(I.getArgOperand(0));
6505 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6509 /// See if we can lower a binary floating-point operation into an SDNode with
6510 /// the specified Opcode. If so, return true and lower it. Otherwise return
6511 /// false, and it will be lowered like a normal call.
6512 /// The caller already checked that \p I calls the appropriate LibFunc with a
6513 /// correct prototype.
6514 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6516 // We already checked this call's prototype; verify it doesn't modify errno.
6517 if (!I.onlyReadsMemory())
6520 SDValue Tmp0 = getValue(I.getArgOperand(0));
6521 SDValue Tmp1 = getValue(I.getArgOperand(1));
6522 EVT VT = Tmp0.getValueType();
6523 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6527 void SelectionDAGBuilder::visitCall(const CallInst &I) {
6528 // Handle inline assembly differently.
6529 if (isa<InlineAsm>(I.getCalledValue())) {
6534 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6535 computeUsesVAFloatArgument(I, MMI);
6537 const char *RenameFn = nullptr;
6538 if (Function *F = I.getCalledFunction()) {
6539 if (F->isDeclaration()) {
6540 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
6541 if (unsigned IID = II->getIntrinsicID(F)) {
6542 RenameFn = visitIntrinsicCall(I, IID);
6547 if (Intrinsic::ID IID = F->getIntrinsicID()) {
6548 RenameFn = visitIntrinsicCall(I, IID);
6554 // Check for well-known libc/libm calls. If the function is internal, it
6555 // can't be a library call. Don't do the check if marked as nobuiltin for
6558 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
6559 LibInfo->getLibFunc(*F, Func) &&
6560 LibInfo->hasOptimizedCodeGen(Func)) {
6563 case LibFunc_copysign:
6564 case LibFunc_copysignf:
6565 case LibFunc_copysignl:
6566 // We already checked this call's prototype; verify it doesn't modify
6568 if (I.onlyReadsMemory()) {
6569 SDValue LHS = getValue(I.getArgOperand(0));
6570 SDValue RHS = getValue(I.getArgOperand(1));
6571 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
6572 LHS.getValueType(), LHS, RHS));
6579 if (visitUnaryFloatCall(I, ISD::FABS))
6585 if (visitBinaryFloatCall(I, ISD::FMINNUM))
6591 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
6597 if (visitUnaryFloatCall(I, ISD::FSIN))
6603 if (visitUnaryFloatCall(I, ISD::FCOS))
6609 case LibFunc_sqrt_finite:
6610 case LibFunc_sqrtf_finite:
6611 case LibFunc_sqrtl_finite:
6612 if (visitUnaryFloatCall(I, ISD::FSQRT))
6616 case LibFunc_floorf:
6617 case LibFunc_floorl:
6618 if (visitUnaryFloatCall(I, ISD::FFLOOR))
6621 case LibFunc_nearbyint:
6622 case LibFunc_nearbyintf:
6623 case LibFunc_nearbyintl:
6624 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
6630 if (visitUnaryFloatCall(I, ISD::FCEIL))
6636 if (visitUnaryFloatCall(I, ISD::FRINT))
6640 case LibFunc_roundf:
6641 case LibFunc_roundl:
6642 if (visitUnaryFloatCall(I, ISD::FROUND))
6646 case LibFunc_truncf:
6647 case LibFunc_truncl:
6648 if (visitUnaryFloatCall(I, ISD::FTRUNC))
6654 if (visitUnaryFloatCall(I, ISD::FLOG2))
6660 if (visitUnaryFloatCall(I, ISD::FEXP2))
6663 case LibFunc_memcmp:
6664 if (visitMemCmpCall(I))
6667 case LibFunc_mempcpy:
6668 if (visitMemPCpyCall(I))
6671 case LibFunc_memchr:
6672 if (visitMemChrCall(I))
6675 case LibFunc_strcpy:
6676 if (visitStrCpyCall(I, false))
6679 case LibFunc_stpcpy:
6680 if (visitStrCpyCall(I, true))
6683 case LibFunc_strcmp:
6684 if (visitStrCmpCall(I))
6687 case LibFunc_strlen:
6688 if (visitStrLenCall(I))
6691 case LibFunc_strnlen:
6692 if (visitStrNLenCall(I))
6701 Callee = getValue(I.getCalledValue());
6703 Callee = DAG.getExternalSymbol(
6705 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
6707 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
6708 // have to do anything here to lower funclet bundles.
6709 assert(!I.hasOperandBundlesOtherThan(
6710 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
6711 "Cannot lower calls with arbitrary operand bundles!");
6713 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
6714 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
6716 // Check if we can potentially perform a tail call. More detailed checking
6717 // is be done within LowerCallTo, after more information about the call is
6719 LowerCallTo(&I, Callee, I.isTailCall());
6724 /// AsmOperandInfo - This contains information for each constraint that we are
6726 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
6728 /// CallOperand - If this is the result output operand or a clobber
6729 /// this is null, otherwise it is the incoming operand to the CallInst.
6730 /// This gets modified as the asm is processed.
6731 SDValue CallOperand;
6733 /// AssignedRegs - If this is a register or register class operand, this
6734 /// contains the set of register corresponding to the operand.
6735 RegsForValue AssignedRegs;
6737 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
6738 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
6741 /// Whether or not this operand accesses memory
6742 bool hasMemory(const TargetLowering &TLI) const {
6743 // Indirect operand accesses access memory.
6747 for (const auto &Code : Codes)
6748 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
6754 /// getCallOperandValEVT - Return the EVT of the Value* that this operand
6755 /// corresponds to. If there is no Value* for this operand, it returns
6757 EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
6758 const DataLayout &DL) const {
6759 if (!CallOperandVal) return MVT::Other;
6761 if (isa<BasicBlock>(CallOperandVal))
6762 return TLI.getPointerTy(DL);
6764 llvm::Type *OpTy = CallOperandVal->getType();
6766 // FIXME: code duplicated from TargetLowering::ParseConstraints().
6767 // If this is an indirect operand, the operand is a pointer to the
6770 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
6772 report_fatal_error("Indirect operand for inline asm not a pointer!");
6773 OpTy = PtrTy->getElementType();
6776 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6777 if (StructType *STy = dyn_cast<StructType>(OpTy))
6778 if (STy->getNumElements() == 1)
6779 OpTy = STy->getElementType(0);
6781 // If OpTy is not a single value, it may be a struct/union that we
6782 // can tile with integers.
6783 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6784 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
6793 OpTy = IntegerType::get(Context, BitSize);
6798 return TLI.getValueType(DL, OpTy, true);
6802 typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
6804 } // end anonymous namespace
6806 /// Make sure that the output operand \p OpInfo and its corresponding input
6807 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
6809 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
6810 SDISelAsmOperandInfo &MatchingOpInfo,
6811 SelectionDAG &DAG) {
6812 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
6815 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
6816 const auto &TLI = DAG.getTargetLoweringInfo();
6818 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6819 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
6820 OpInfo.ConstraintVT);
6821 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6822 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
6823 MatchingOpInfo.ConstraintVT);
6824 if ((OpInfo.ConstraintVT.isInteger() !=
6825 MatchingOpInfo.ConstraintVT.isInteger()) ||
6826 (MatchRC.second != InputRC.second)) {
6827 // FIXME: error out in a more elegant fashion
6828 report_fatal_error("Unsupported asm: input constraint"
6829 " with a matching output constraint of"
6830 " incompatible type!");
6832 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
6835 /// Get a direct memory input to behave well as an indirect operand.
6836 /// This may introduce stores, hence the need for a \p Chain.
6837 /// \return The (possibly updated) chain.
6838 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
6839 SDISelAsmOperandInfo &OpInfo,
6840 SelectionDAG &DAG) {
6841 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6843 // If we don't have an indirect input, put it in the constpool if we can,
6844 // otherwise spill it to a stack slot.
6845 // TODO: This isn't quite right. We need to handle these according to
6846 // the addressing mode that the constraint wants. Also, this may take
6847 // an additional register for the computation and we don't want that
6850 // If the operand is a float, integer, or vector constant, spill to a
6851 // constant pool entry to get its address.
6852 const Value *OpVal = OpInfo.CallOperandVal;
6853 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
6854 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
6855 OpInfo.CallOperand = DAG.getConstantPool(
6856 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
6860 // Otherwise, create a stack slot and emit a store to it before the asm.
6861 Type *Ty = OpVal->getType();
6862 auto &DL = DAG.getDataLayout();
6863 uint64_t TySize = DL.getTypeAllocSize(Ty);
6864 unsigned Align = DL.getPrefTypeAlignment(Ty);
6865 MachineFunction &MF = DAG.getMachineFunction();
6866 int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
6867 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
6868 Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
6869 MachinePointerInfo::getFixedStack(MF, SSFI));
6870 OpInfo.CallOperand = StackSlot;
6875 /// GetRegistersForValue - Assign registers (virtual or physical) for the
6876 /// specified operand. We prefer to assign virtual registers, to allow the
6877 /// register allocator to handle the assignment process. However, if the asm
6878 /// uses features that we can't model on machineinstrs, we have SDISel do the
6879 /// allocation. This produces generally horrible, but correct, code.
6881 /// OpInfo describes the operand.
6883 static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
6885 SDISelAsmOperandInfo &OpInfo) {
6886 LLVMContext &Context = *DAG.getContext();
6888 MachineFunction &MF = DAG.getMachineFunction();
6889 SmallVector<unsigned, 4> Regs;
6890 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
6892 // If this is a constraint for a single physreg, or a constraint for a
6893 // register class, find it.
6894 std::pair<unsigned, const TargetRegisterClass *> PhysReg =
6895 TLI.getRegForInlineAsmConstraint(&TRI, OpInfo.ConstraintCode,
6896 OpInfo.ConstraintVT);
6898 unsigned NumRegs = 1;
6899 if (OpInfo.ConstraintVT != MVT::Other) {
6900 // If this is a FP input in an integer register (or visa versa) insert a bit
6901 // cast of the input value. More generally, handle any case where the input
6902 // value disagrees with the register class we plan to stick this in.
6903 if (OpInfo.Type == InlineAsm::isInput && PhysReg.second &&
6904 !TRI.isTypeLegalForClass(*PhysReg.second, OpInfo.ConstraintVT)) {
6905 // Try to convert to the first EVT that the reg class contains. If the
6906 // types are identical size, use a bitcast to convert (e.g. two differing
6908 MVT RegVT = *TRI.legalclasstypes_begin(*PhysReg.second);
6909 if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
6910 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6911 RegVT, OpInfo.CallOperand);
6912 OpInfo.ConstraintVT = RegVT;
6913 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
6914 // If the input is a FP value and we want it in FP registers, do a
6915 // bitcast to the corresponding integer type. This turns an f64 value
6916 // into i64, which can be passed with two i32 values on a 32-bit
6918 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
6919 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6920 RegVT, OpInfo.CallOperand);
6921 OpInfo.ConstraintVT = RegVT;
6925 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
6929 EVT ValueVT = OpInfo.ConstraintVT;
6931 // If this is a constraint for a specific physical register, like {r17},
6933 if (unsigned AssignedReg = PhysReg.first) {
6934 const TargetRegisterClass *RC = PhysReg.second;
6935 if (OpInfo.ConstraintVT == MVT::Other)
6936 ValueVT = *TRI.legalclasstypes_begin(*RC);
6938 // Get the actual register value type. This is important, because the user
6939 // may have asked for (e.g.) the AX register in i32 type. We need to
6940 // remember that AX is actually i16 to get the right extension.
6941 RegVT = *TRI.legalclasstypes_begin(*RC);
6943 // This is a explicit reference to a physical register.
6944 Regs.push_back(AssignedReg);
6946 // If this is an expanded reference, add the rest of the regs to Regs.
6948 TargetRegisterClass::iterator I = RC->begin();
6949 for (; *I != AssignedReg; ++I)
6950 assert(I != RC->end() && "Didn't find reg!");
6952 // Already added the first reg.
6954 for (; NumRegs; --NumRegs, ++I) {
6955 assert(I != RC->end() && "Ran out of registers to allocate!");
6960 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6964 // Otherwise, if this was a reference to an LLVM register class, create vregs
6965 // for this reference.
6966 if (const TargetRegisterClass *RC = PhysReg.second) {
6967 RegVT = *TRI.legalclasstypes_begin(*RC);
6968 if (OpInfo.ConstraintVT == MVT::Other)
6971 // Create the appropriate number of virtual registers.
6972 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6973 for (; NumRegs; --NumRegs)
6974 Regs.push_back(RegInfo.createVirtualRegister(RC));
6976 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6980 // Otherwise, we couldn't allocate enough registers for this.
6984 findMatchingInlineAsmOperand(unsigned OperandNo,
6985 const std::vector<SDValue> &AsmNodeOperands) {
6986 // Scan until we find the definition we already emitted of this operand.
6987 unsigned CurOp = InlineAsm::Op_FirstOperand;
6988 for (; OperandNo; --OperandNo) {
6989 // Advance to the next operand.
6991 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6992 assert((InlineAsm::isRegDefKind(OpFlag) ||
6993 InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
6994 InlineAsm::isMemKind(OpFlag)) &&
6995 "Skipped past definitions?");
6996 CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
7001 /// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
7002 /// \return true if it has succeeded, false otherwise
7003 static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
7004 MVT RegVT, SelectionDAG &DAG) {
7005 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7006 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
7007 for (unsigned i = 0, e = NumRegs; i != e; ++i) {
7008 if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
7009 Regs.push_back(RegInfo.createVirtualRegister(RC));
7020 explicit ExtraFlags(ImmutableCallSite CS) {
7021 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7022 if (IA->hasSideEffects())
7023 Flags |= InlineAsm::Extra_HasSideEffects;
7024 if (IA->isAlignStack())
7025 Flags |= InlineAsm::Extra_IsAlignStack;
7026 if (CS.isConvergent())
7027 Flags |= InlineAsm::Extra_IsConvergent;
7028 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
7031 void update(const llvm::TargetLowering::AsmOperandInfo &OpInfo) {
7032 // Ideally, we would only check against memory constraints. However, the
7033 // meaning of an Other constraint can be target-specific and we can't easily
7034 // reason about it. Therefore, be conservative and set MayLoad/MayStore
7035 // for Other constraints as well.
7036 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
7037 OpInfo.ConstraintType == TargetLowering::C_Other) {
7038 if (OpInfo.Type == InlineAsm::isInput)
7039 Flags |= InlineAsm::Extra_MayLoad;
7040 else if (OpInfo.Type == InlineAsm::isOutput)
7041 Flags |= InlineAsm::Extra_MayStore;
7042 else if (OpInfo.Type == InlineAsm::isClobber)
7043 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
7047 unsigned get() const { return Flags; }
7050 /// visitInlineAsm - Handle a call to an InlineAsm object.
7052 void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
7053 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7055 /// ConstraintOperands - Information about all of the constraints.
7056 SDISelAsmOperandInfoVector ConstraintOperands;
7058 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7059 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
7060 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
7062 bool hasMemory = false;
7064 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7065 ExtraFlags ExtraInfo(CS);
7067 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
7068 unsigned ResNo = 0; // ResNo - The result number of the next output.
7069 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
7070 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
7071 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
7073 MVT OpVT = MVT::Other;
7075 // Compute the value type for each operand.
7076 if (OpInfo.Type == InlineAsm::isInput ||
7077 (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
7078 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
7080 // Process the call argument. BasicBlocks are labels, currently appearing
7082 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
7083 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
7085 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
7090 .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
7094 if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
7095 // The return value of the call is this value. As such, there is no
7096 // corresponding argument.
7097 assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7098 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
7099 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
7100 STy->getElementType(ResNo));
7102 assert(ResNo == 0 && "Asm only has one result!");
7103 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
7108 OpInfo.ConstraintVT = OpVT;
7111 hasMemory = OpInfo.hasMemory(TLI);
7113 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
7114 // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
7115 auto TargetConstraint = TargetConstraints[i];
7117 // Compute the constraint code and ConstraintType to use.
7118 TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
7120 ExtraInfo.update(TargetConstraint);
7123 SDValue Chain, Flag;
7125 // We won't need to flush pending loads if this asm doesn't touch
7126 // memory and is nonvolatile.
7127 if (hasMemory || IA->hasSideEffects())
7130 Chain = DAG.getRoot();
7132 // Second pass over the constraints: compute which constraint option to use
7133 // and assign registers to constraints that want a specific physreg.
7134 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7135 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7137 // If this is an output operand with a matching input operand, look up the
7138 // matching input. If their types mismatch, e.g. one is an integer, the
7139 // other is floating point, or their sizes are different, flag it as an
7141 if (OpInfo.hasMatchingInput()) {
7142 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
7143 patchMatchingInput(OpInfo, Input, DAG);
7146 // Compute the constraint code and ConstraintType to use.
7147 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
7149 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7150 OpInfo.Type == InlineAsm::isClobber)
7153 // If this is a memory input, and if the operand is not indirect, do what we
7154 // need to to provide an address for the memory input.
7155 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7156 !OpInfo.isIndirect) {
7157 assert((OpInfo.isMultipleAlternative ||
7158 (OpInfo.Type == InlineAsm::isInput)) &&
7159 "Can only indirectify direct input operands!");
7161 // Memory operands really want the address of the value.
7162 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
7164 // There is no longer a Value* corresponding to this operand.
7165 OpInfo.CallOperandVal = nullptr;
7167 // It is now an indirect operand.
7168 OpInfo.isIndirect = true;
7171 // If this constraint is for a specific register, allocate it before
7173 if (OpInfo.ConstraintType == TargetLowering::C_Register)
7174 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
7177 // Third pass - Loop over all of the operands, assigning virtual or physregs
7178 // to register class operands.
7179 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7180 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7182 // C_Register operands have already been allocated, Other/Memory don't need
7184 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
7185 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
7188 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
7189 std::vector<SDValue> AsmNodeOperands;
7190 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
7191 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
7192 IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
7194 // If we have a !srcloc metadata node associated with it, we want to attach
7195 // this to the ultimately generated inline asm machineinstr. To do this, we
7196 // pass in the third operand as this (potentially null) inline asm MDNode.
7197 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
7198 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
7200 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7201 // bits as operand 3.
7202 AsmNodeOperands.push_back(DAG.getTargetConstant(
7203 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7205 // Loop over all of the inputs, copying the operand values into the
7206 // appropriate registers and processing the output regs.
7207 RegsForValue RetValRegs;
7209 // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
7210 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
7212 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7213 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7215 switch (OpInfo.Type) {
7216 case InlineAsm::isOutput: {
7217 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
7218 OpInfo.ConstraintType != TargetLowering::C_Register) {
7219 // Memory output, or 'other' output (e.g. 'X' constraint).
7220 assert(OpInfo.isIndirect && "Memory output must be indirect operand");
7222 unsigned ConstraintID =
7223 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7224 assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7225 "Failed to convert memory constraint code to constraint id.");
7227 // Add information to the INLINEASM node to know about this output.
7228 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7229 OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7230 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7232 AsmNodeOperands.push_back(OpInfo.CallOperand);
7236 // Otherwise, this is a register or register class output.
7238 // Copy the output from the appropriate register. Find a register that
7240 if (OpInfo.AssignedRegs.Regs.empty()) {
7242 CS, "couldn't allocate output register for constraint '" +
7243 Twine(OpInfo.ConstraintCode) + "'");
7247 // If this is an indirect operand, store through the pointer after the
7249 if (OpInfo.isIndirect) {
7250 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7251 OpInfo.CallOperandVal));
7253 // This is the result value of the call.
7254 assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
7255 // Concatenate this output onto the outputs list.
7256 RetValRegs.append(OpInfo.AssignedRegs);
7259 // Add information to the INLINEASM node to know that this register is
7262 .AddInlineAsmOperands(OpInfo.isEarlyClobber
7263 ? InlineAsm::Kind_RegDefEarlyClobber
7264 : InlineAsm::Kind_RegDef,
7265 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7268 case InlineAsm::isInput: {
7269 SDValue InOperandVal = OpInfo.CallOperand;
7271 if (OpInfo.isMatchingInputConstraint()) {
7272 // If this is required to match an output register we have already set,
7273 // just use its register.
7274 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7277 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7278 if (InlineAsm::isRegDefKind(OpFlag) ||
7279 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7280 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7281 if (OpInfo.isIndirect) {
7282 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7283 emitInlineAsmError(CS, "inline asm not supported yet:"
7284 " don't know how to handle tied "
7285 "indirect register inputs");
7289 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7290 SmallVector<unsigned, 4> Regs;
7292 if (!createVirtualRegs(Regs,
7293 InlineAsm::getNumOperandRegisters(OpFlag),
7295 emitInlineAsmError(CS, "inline asm error: This value type register "
7296 "class is not natively supported!");
7300 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7302 SDLoc dl = getCurSDLoc();
7303 // Use the produced MatchedRegs object to
7304 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
7305 CS.getInstruction());
7306 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7307 true, OpInfo.getMatchedOperand(), dl,
7308 DAG, AsmNodeOperands);
7312 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
7313 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
7314 "Unexpected number of operands");
7315 // Add information to the INLINEASM node to know about this input.
7316 // See InlineAsm.h isUseOperandTiedToDef.
7317 OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7318 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7319 OpInfo.getMatchedOperand());
7320 AsmNodeOperands.push_back(DAG.getTargetConstant(
7321 OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7322 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7326 // Treat indirect 'X' constraint as memory.
7327 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7329 OpInfo.ConstraintType = TargetLowering::C_Memory;
7331 if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7332 std::vector<SDValue> Ops;
7333 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7336 emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7337 Twine(OpInfo.ConstraintCode) + "'");
7341 // Add information to the INLINEASM node to know about this input.
7342 unsigned ResOpType =
7343 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7344 AsmNodeOperands.push_back(DAG.getTargetConstant(
7345 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7346 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7350 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7351 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
7352 assert(InOperandVal.getValueType() ==
7353 TLI.getPointerTy(DAG.getDataLayout()) &&
7354 "Memory operands expect pointer values");
7356 unsigned ConstraintID =
7357 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7358 assert(ConstraintID != InlineAsm::Constraint_Unknown &&
7359 "Failed to convert memory constraint code to constraint id.");
7361 // Add information to the INLINEASM node to know about this input.
7362 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7363 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7364 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7367 AsmNodeOperands.push_back(InOperandVal);
7371 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
7372 OpInfo.ConstraintType == TargetLowering::C_Register) &&
7373 "Unknown constraint type!");
7375 // TODO: Support this.
7376 if (OpInfo.isIndirect) {
7378 CS, "Don't know how to handle indirect register inputs yet "
7379 "for constraint '" +
7380 Twine(OpInfo.ConstraintCode) + "'");
7384 // Copy the input into the appropriate registers.
7385 if (OpInfo.AssignedRegs.Regs.empty()) {
7386 emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7387 Twine(OpInfo.ConstraintCode) + "'");
7391 SDLoc dl = getCurSDLoc();
7393 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7394 Chain, &Flag, CS.getInstruction());
7396 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7397 dl, DAG, AsmNodeOperands);
7400 case InlineAsm::isClobber: {
7401 // Add the clobbered value to the operand list, so that the register
7402 // allocator is aware that the physreg got clobbered.
7403 if (!OpInfo.AssignedRegs.Regs.empty())
7404 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7405 false, 0, getCurSDLoc(), DAG,
7412 // Finish up input operands. Set the input chain and add the flag last.
7413 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7414 if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7416 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7417 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7418 Flag = Chain.getValue(1);
7420 // If this asm returns a register value, copy the result from that register
7421 // and set it as the value of the call.
7422 if (!RetValRegs.Regs.empty()) {
7423 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7424 Chain, &Flag, CS.getInstruction());
7426 // FIXME: Why don't we do this for inline asms with MRVs?
7427 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
7428 EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7430 // If any of the results of the inline asm is a vector, it may have the
7431 // wrong width/num elts. This can happen for register classes that can
7432 // contain multiple different value types. The preg or vreg allocated may
7433 // not have the same VT as was expected. Convert it to the right type
7434 // with bit_convert.
7435 if (ResultType != Val.getValueType() && Val.getValueType().isVector()) {
7436 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(),
7439 } else if (ResultType != Val.getValueType() &&
7440 ResultType.isInteger() && Val.getValueType().isInteger()) {
7441 // If a result value was tied to an input value, the computed result may
7442 // have a wider width than the expected result. Extract the relevant
7444 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
7447 assert(ResultType == Val.getValueType() && "Asm result value mismatch!");
7450 setValue(CS.getInstruction(), Val);
7451 // Don't need to use this as a chain in this case.
7452 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7456 std::vector<std::pair<SDValue, const Value *> > StoresToEmit;
7458 // Process indirect outputs, first output all of the flagged copies out of
7460 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7461 RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7462 const Value *Ptr = IndirectStoresToEmit[i].second;
7463 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7465 StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7468 // Emit the non-flagged stores from the physregs.
7469 SmallVector<SDValue, 8> OutChains;
7470 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7471 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7472 getValue(StoresToEmit[i].second),
7473 MachinePointerInfo(StoresToEmit[i].second));
7474 OutChains.push_back(Val);
7477 if (!OutChains.empty())
7478 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7483 void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7484 const Twine &Message) {
7485 LLVMContext &Ctx = *DAG.getContext();
7486 Ctx.emitError(CS.getInstruction(), Message);
7488 // Make sure we leave the DAG in a valid state
7489 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7490 auto VT = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7491 setValue(CS.getInstruction(), DAG.getUNDEF(VT));
7494 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7495 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
7496 MVT::Other, getRoot(),
7497 getValue(I.getArgOperand(0)),
7498 DAG.getSrcValue(I.getArgOperand(0))));
7501 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7502 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7503 const DataLayout &DL = DAG.getDataLayout();
7504 SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7505 getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7506 DAG.getSrcValue(I.getOperand(0)),
7507 DL.getABITypeAlignment(I.getType()));
7509 DAG.setRoot(V.getValue(1));
7512 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
7513 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
7514 MVT::Other, getRoot(),
7515 getValue(I.getArgOperand(0)),
7516 DAG.getSrcValue(I.getArgOperand(0))));
7519 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
7520 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
7521 MVT::Other, getRoot(),
7522 getValue(I.getArgOperand(0)),
7523 getValue(I.getArgOperand(1)),
7524 DAG.getSrcValue(I.getArgOperand(0)),
7525 DAG.getSrcValue(I.getArgOperand(1))));
7528 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
7529 const Instruction &I,
7531 const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
7535 ConstantRange CR = getConstantRangeFromMetadata(*Range);
7536 if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
7539 APInt Lo = CR.getUnsignedMin();
7540 if (!Lo.isMinValue())
7543 APInt Hi = CR.getUnsignedMax();
7544 unsigned Bits = Hi.getActiveBits();
7546 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
7548 SDLoc SL = getCurSDLoc();
7550 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
7551 DAG.getValueType(SmallVT));
7552 unsigned NumVals = Op.getNode()->getNumValues();
7556 SmallVector<SDValue, 4> Ops;
7558 Ops.push_back(ZExt);
7559 for (unsigned I = 1; I != NumVals; ++I)
7560 Ops.push_back(Op.getValue(I));
7562 return DAG.getMergeValues(Ops, SL);
7565 /// \brief Populate a CallLowerinInfo (into \p CLI) based on the properties of
7566 /// the call being lowered.
7568 /// This is a helper for lowering intrinsics that follow a target calling
7569 /// convention or require stack pointer adjustment. Only a subset of the
7570 /// intrinsic's operands need to participate in the calling convention.
7571 void SelectionDAGBuilder::populateCallLoweringInfo(
7572 TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
7573 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
7574 bool IsPatchPoint) {
7575 TargetLowering::ArgListTy Args;
7576 Args.reserve(NumArgs);
7578 // Populate the argument list.
7579 // Attributes for args start at offset 1, after the return attribute.
7580 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
7581 ArgI != ArgE; ++ArgI) {
7582 const Value *V = CS->getOperand(ArgI);
7584 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
7586 TargetLowering::ArgListEntry Entry;
7587 Entry.Node = getValue(V);
7588 Entry.Ty = V->getType();
7589 Entry.setAttributes(&CS, ArgIdx);
7590 Args.push_back(Entry);
7593 CLI.setDebugLoc(getCurSDLoc())
7594 .setChain(getRoot())
7595 .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
7596 .setDiscardResult(CS->use_empty())
7597 .setIsPatchPoint(IsPatchPoint);
7600 /// \brief Add a stack map intrinsic call's live variable operands to a stackmap
7601 /// or patchpoint target node's operand list.
7603 /// Constants are converted to TargetConstants purely as an optimization to
7604 /// avoid constant materialization and register allocation.
7606 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
7607 /// generate addess computation nodes, and so ExpandISelPseudo can convert the
7608 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
7609 /// address materialization and register allocation, but may also be required
7610 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
7611 /// alloca in the entry block, then the runtime may assume that the alloca's
7612 /// StackMap location can be read immediately after compilation and that the
7613 /// location is valid at any point during execution (this is similar to the
7614 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
7615 /// only available in a register, then the runtime would need to trap when
7616 /// execution reaches the StackMap in order to read the alloca's location.
7617 static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
7618 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
7619 SelectionDAGBuilder &Builder) {
7620 for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
7621 SDValue OpVal = Builder.getValue(CS.getArgument(i));
7622 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
7624 Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
7626 Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
7627 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
7628 const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
7629 Ops.push_back(Builder.DAG.getTargetFrameIndex(
7630 FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
7632 Ops.push_back(OpVal);
7636 /// \brief Lower llvm.experimental.stackmap directly to its target opcode.
7637 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
7638 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
7639 // [live variables...])
7641 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
7643 SDValue Chain, InFlag, Callee, NullPtr;
7644 SmallVector<SDValue, 32> Ops;
7646 SDLoc DL = getCurSDLoc();
7647 Callee = getValue(CI.getCalledValue());
7648 NullPtr = DAG.getIntPtrConstant(0, DL, true);
7650 // The stackmap intrinsic only records the live variables (the arguemnts
7651 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
7652 // intrinsic, this won't be lowered to a function call. This means we don't
7653 // have to worry about calling conventions and target specific lowering code.
7654 // Instead we perform the call lowering right here.
7656 // chain, flag = CALLSEQ_START(chain, 0, 0)
7657 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
7658 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
7660 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
7661 InFlag = Chain.getValue(1);
7663 // Add the <id> and <numBytes> constants.
7664 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
7665 Ops.push_back(DAG.getTargetConstant(
7666 cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
7667 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
7668 Ops.push_back(DAG.getTargetConstant(
7669 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
7672 // Push live variables for the stack map.
7673 addStackMapLiveVars(&CI, 2, DL, Ops, *this);
7675 // We are not pushing any register mask info here on the operands list,
7676 // because the stackmap doesn't clobber anything.
7678 // Push the chain and the glue flag.
7679 Ops.push_back(Chain);
7680 Ops.push_back(InFlag);
7682 // Create the STACKMAP node.
7683 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7684 SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
7685 Chain = SDValue(SM, 0);
7686 InFlag = Chain.getValue(1);
7688 Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
7690 // Stackmaps don't generate values, so nothing goes into the NodeMap.
7692 // Set the root to the target-lowered call chain.
7695 // Inform the Frame Information that we have a stackmap in this function.
7696 FuncInfo.MF->getFrameInfo().setHasStackMap();
7699 /// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
7700 void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
7701 const BasicBlock *EHPadBB) {
7702 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
7707 // [live variables...])
7709 CallingConv::ID CC = CS.getCallingConv();
7710 bool IsAnyRegCC = CC == CallingConv::AnyReg;
7711 bool HasDef = !CS->getType()->isVoidTy();
7712 SDLoc dl = getCurSDLoc();
7713 SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
7715 // Handle immediate and symbolic callees.
7716 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
7717 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
7719 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
7720 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
7721 SDLoc(SymbolicCallee),
7722 SymbolicCallee->getValueType(0));
7724 // Get the real number of arguments participating in the call <numArgs>
7725 SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
7726 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
7728 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
7729 // Intrinsics include all meta-operands up to but not including CC.
7730 unsigned NumMetaOpers = PatchPointOpers::CCPos;
7731 assert(CS.arg_size() >= NumMetaOpers + NumArgs &&
7732 "Not enough arguments provided to the patchpoint intrinsic");
7734 // For AnyRegCC the arguments are lowered later on manually.
7735 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
7737 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
7739 TargetLowering::CallLoweringInfo CLI(DAG);
7740 populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
7742 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7744 SDNode *CallEnd = Result.second.getNode();
7745 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
7746 CallEnd = CallEnd->getOperand(0).getNode();
7748 /// Get a call instruction from the call sequence chain.
7749 /// Tail calls are not allowed.
7750 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
7751 "Expected a callseq node.");
7752 SDNode *Call = CallEnd->getOperand(0).getNode();
7753 bool HasGlue = Call->getGluedNode();
7755 // Replace the target specific call node with the patchable intrinsic.
7756 SmallVector<SDValue, 8> Ops;
7758 // Add the <id> and <numBytes> constants.
7759 SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
7760 Ops.push_back(DAG.getTargetConstant(
7761 cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
7762 SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
7763 Ops.push_back(DAG.getTargetConstant(
7764 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
7768 Ops.push_back(Callee);
7770 // Adjust <numArgs> to account for any arguments that have been passed on the
7772 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
7773 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
7774 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
7775 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
7777 // Add the calling convention
7778 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
7780 // Add the arguments we omitted previously. The register allocator should
7781 // place these in any free register.
7783 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
7784 Ops.push_back(getValue(CS.getArgument(i)));
7786 // Push the arguments from the call instruction up to the register mask.
7787 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
7788 Ops.append(Call->op_begin() + 2, e);
7790 // Push live variables for the stack map.
7791 addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
7793 // Push the register mask info.
7795 Ops.push_back(*(Call->op_end()-2));
7797 Ops.push_back(*(Call->op_end()-1));
7799 // Push the chain (this is originally the first operand of the call, but
7800 // becomes now the last or second to last operand).
7801 Ops.push_back(*(Call->op_begin()));
7803 // Push the glue flag (last operand).
7805 Ops.push_back(*(Call->op_end()-1));
7808 if (IsAnyRegCC && HasDef) {
7809 // Create the return types based on the intrinsic definition
7810 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7811 SmallVector<EVT, 3> ValueVTs;
7812 ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7813 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
7815 // There is always a chain and a glue type at the end
7816 ValueVTs.push_back(MVT::Other);
7817 ValueVTs.push_back(MVT::Glue);
7818 NodeTys = DAG.getVTList(ValueVTs);
7820 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7822 // Replace the target specific call node with a PATCHPOINT node.
7823 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
7826 // Update the NodeMap.
7829 setValue(CS.getInstruction(), SDValue(MN, 0));
7831 setValue(CS.getInstruction(), Result.first);
7834 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
7835 // call sequence. Furthermore the location of the chain and glue can change
7836 // when the AnyReg calling convention is used and the intrinsic returns a
7838 if (IsAnyRegCC && HasDef) {
7839 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
7840 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
7841 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
7843 DAG.ReplaceAllUsesWith(Call, MN);
7844 DAG.DeleteNode(Call);
7846 // Inform the Frame Information that we have a patchpoint in this function.
7847 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
7850 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
7851 unsigned Intrinsic) {
7852 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7853 SDValue Op1 = getValue(I.getArgOperand(0));
7855 if (I.getNumArgOperands() > 1)
7856 Op2 = getValue(I.getArgOperand(1));
7857 SDLoc dl = getCurSDLoc();
7858 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7861 if (isa<FPMathOperator>(I))
7862 FMF = I.getFastMathFlags();
7863 SDNodeFlags SDFlags;
7864 SDFlags.setNoNaNs(FMF.noNaNs());
7866 switch (Intrinsic) {
7867 case Intrinsic::experimental_vector_reduce_fadd:
7868 if (FMF.unsafeAlgebra())
7869 Res = DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2);
7871 Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2);
7873 case Intrinsic::experimental_vector_reduce_fmul:
7874 if (FMF.unsafeAlgebra())
7875 Res = DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2);
7877 Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2);
7879 case Intrinsic::experimental_vector_reduce_add:
7880 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
7882 case Intrinsic::experimental_vector_reduce_mul:
7883 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
7885 case Intrinsic::experimental_vector_reduce_and:
7886 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
7888 case Intrinsic::experimental_vector_reduce_or:
7889 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
7891 case Intrinsic::experimental_vector_reduce_xor:
7892 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
7894 case Intrinsic::experimental_vector_reduce_smax:
7895 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
7897 case Intrinsic::experimental_vector_reduce_smin:
7898 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
7900 case Intrinsic::experimental_vector_reduce_umax:
7901 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
7903 case Intrinsic::experimental_vector_reduce_umin:
7904 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
7906 case Intrinsic::experimental_vector_reduce_fmax: {
7907 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
7910 case Intrinsic::experimental_vector_reduce_fmin: {
7911 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
7915 llvm_unreachable("Unhandled vector reduce intrinsic");
7920 /// Returns an AttributeList representing the attributes applied to the return
7921 /// value of the given call.
7922 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
7923 SmallVector<Attribute::AttrKind, 2> Attrs;
7925 Attrs.push_back(Attribute::SExt);
7927 Attrs.push_back(Attribute::ZExt);
7929 Attrs.push_back(Attribute::InReg);
7931 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
7935 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
7936 /// implementation, which just calls LowerCall.
7937 /// FIXME: When all targets are
7938 /// migrated to using LowerCall, this hook should be integrated into SDISel.
7939 std::pair<SDValue, SDValue>
7940 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
7941 // Handle the incoming return values from the call.
7943 Type *OrigRetTy = CLI.RetTy;
7944 SmallVector<EVT, 4> RetTys;
7945 SmallVector<uint64_t, 4> Offsets;
7946 auto &DL = CLI.DAG.getDataLayout();
7947 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
7949 if (CLI.IsPostTypeLegalization) {
7950 // If we are lowering a libcall after legalization, split the return type.
7951 SmallVector<EVT, 4> OldRetTys = std::move(RetTys);
7952 SmallVector<uint64_t, 4> OldOffsets = std::move(Offsets);
7953 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
7954 EVT RetVT = OldRetTys[i];
7955 uint64_t Offset = OldOffsets[i];
7956 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
7957 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
7958 unsigned RegisterVTSize = RegisterVT.getSizeInBits();
7959 RetTys.append(NumRegs, RegisterVT);
7960 for (unsigned j = 0; j != NumRegs; ++j)
7961 Offsets.push_back(Offset + j * RegisterVTSize);
7965 SmallVector<ISD::OutputArg, 4> Outs;
7966 GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
7968 bool CanLowerReturn =
7969 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
7970 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
7972 SDValue DemoteStackSlot;
7973 int DemoteStackIdx = -100;
7974 if (!CanLowerReturn) {
7975 // FIXME: equivalent assert?
7976 // assert(!CS.hasInAllocaArgument() &&
7977 // "sret demotion is incompatible with inalloca");
7978 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
7979 unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
7980 MachineFunction &MF = CLI.DAG.getMachineFunction();
7981 DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7982 Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
7984 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
7986 Entry.Node = DemoteStackSlot;
7987 Entry.Ty = StackSlotPtrType;
7988 Entry.IsSExt = false;
7989 Entry.IsZExt = false;
7990 Entry.IsInReg = false;
7991 Entry.IsSRet = true;
7992 Entry.IsNest = false;
7993 Entry.IsByVal = false;
7994 Entry.IsReturned = false;
7995 Entry.IsSwiftSelf = false;
7996 Entry.IsSwiftError = false;
7997 Entry.Alignment = Align;
7998 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
7999 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
8001 // sret demotion isn't compatible with tail-calls, since the sret argument
8002 // points into the callers stack frame.
8003 CLI.IsTailCall = false;
8005 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8008 getRegisterTypeForCallingConv(CLI.RetTy->getContext(), VT);
8010 getNumRegistersForCallingConv(CLI.RetTy->getContext(), VT);
8011 for (unsigned i = 0; i != NumRegs; ++i) {
8012 ISD::InputArg MyFlags;
8013 MyFlags.VT = RegisterVT;
8015 MyFlags.Used = CLI.IsReturnValueUsed;
8017 MyFlags.Flags.setSExt();
8019 MyFlags.Flags.setZExt();
8021 MyFlags.Flags.setInReg();
8022 CLI.Ins.push_back(MyFlags);
8027 // We push in swifterror return as the last element of CLI.Ins.
8028 ArgListTy &Args = CLI.getArgs();
8029 if (supportSwiftError()) {
8030 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8031 if (Args[i].IsSwiftError) {
8032 ISD::InputArg MyFlags;
8033 MyFlags.VT = getPointerTy(DL);
8034 MyFlags.ArgVT = EVT(getPointerTy(DL));
8035 MyFlags.Flags.setSwiftError();
8036 CLI.Ins.push_back(MyFlags);
8041 // Handle all of the outgoing arguments.
8043 CLI.OutVals.clear();
8044 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8045 SmallVector<EVT, 4> ValueVTs;
8046 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
8047 // FIXME: Split arguments if CLI.IsPostTypeLegalization
8048 Type *FinalType = Args[i].Ty;
8049 if (Args[i].IsByVal)
8050 FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
8051 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
8052 FinalType, CLI.CallConv, CLI.IsVarArg);
8053 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
8055 EVT VT = ValueVTs[Value];
8056 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
8057 SDValue Op = SDValue(Args[i].Node.getNode(),
8058 Args[i].Node.getResNo() + Value);
8059 ISD::ArgFlagsTy Flags;
8061 // Certain targets (such as MIPS), may have a different ABI alignment
8062 // for a type depending on the context. Give the target a chance to
8063 // specify the alignment it wants.
8064 unsigned OriginalAlignment = getABIAlignmentForCallingConv(ArgTy, DL);
8070 if (Args[i].IsInReg) {
8071 // If we are using vectorcall calling convention, a structure that is
8072 // passed InReg - is surely an HVA
8073 if (CLI.CallConv == CallingConv::X86_VectorCall &&
8074 isa<StructType>(FinalType)) {
8075 // The first value of a structure is marked
8077 Flags.setHvaStart();
8085 if (Args[i].IsSwiftSelf)
8086 Flags.setSwiftSelf();
8087 if (Args[i].IsSwiftError)
8088 Flags.setSwiftError();
8089 if (Args[i].IsByVal)
8091 if (Args[i].IsInAlloca) {
8092 Flags.setInAlloca();
8093 // Set the byval flag for CCAssignFn callbacks that don't know about
8094 // inalloca. This way we can know how many bytes we should've allocated
8095 // and how many bytes a callee cleanup function will pop. If we port
8096 // inalloca to more targets, we'll have to add custom inalloca handling
8097 // in the various CC lowering callbacks.
8100 if (Args[i].IsByVal || Args[i].IsInAlloca) {
8101 PointerType *Ty = cast<PointerType>(Args[i].Ty);
8102 Type *ElementTy = Ty->getElementType();
8103 Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8104 // For ByVal, alignment should come from FE. BE will guess if this
8105 // info is not there but there are cases it cannot get right.
8106 unsigned FrameAlign;
8107 if (Args[i].Alignment)
8108 FrameAlign = Args[i].Alignment;
8110 FrameAlign = getByValTypeAlignment(ElementTy, DL);
8111 Flags.setByValAlign(FrameAlign);
8116 Flags.setInConsecutiveRegs();
8117 Flags.setOrigAlign(OriginalAlignment);
8119 MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), VT);
8121 getNumRegistersForCallingConv(CLI.RetTy->getContext(), VT);
8122 SmallVector<SDValue, 4> Parts(NumParts);
8123 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
8126 ExtendKind = ISD::SIGN_EXTEND;
8127 else if (Args[i].IsZExt)
8128 ExtendKind = ISD::ZERO_EXTEND;
8130 // Conservatively only handle 'returned' on non-vectors for now
8131 if (Args[i].IsReturned && !Op.getValueType().isVector()) {
8132 assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&
8133 "unexpected use of 'returned'");
8134 // Before passing 'returned' to the target lowering code, ensure that
8135 // either the register MVT and the actual EVT are the same size or that
8136 // the return value and argument are extended in the same way; in these
8137 // cases it's safe to pass the argument register value unchanged as the
8138 // return register value (although it's at the target's option whether
8140 // TODO: allow code generation to take advantage of partially preserved
8141 // registers rather than clobbering the entire register when the
8142 // parameter extension method is not compatible with the return
8144 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
8145 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
8146 CLI.RetZExt == Args[i].IsZExt))
8147 Flags.setReturned();
8150 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
8151 CLI.CS ? CLI.CS->getInstruction() : nullptr, ExtendKind,
8154 for (unsigned j = 0; j != NumParts; ++j) {
8155 // if it isn't first piece, alignment must be 1
8156 ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
8157 i < CLI.NumFixedArgs,
8158 i, j*Parts[j].getValueType().getStoreSize());
8159 if (NumParts > 1 && j == 0)
8160 MyFlags.Flags.setSplit();
8162 MyFlags.Flags.setOrigAlign(1);
8163 if (j == NumParts - 1)
8164 MyFlags.Flags.setSplitEnd();
8167 CLI.Outs.push_back(MyFlags);
8168 CLI.OutVals.push_back(Parts[j]);
8171 if (NeedsRegBlock && Value == NumValues - 1)
8172 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
8176 SmallVector<SDValue, 4> InVals;
8177 CLI.Chain = LowerCall(CLI, InVals);
8179 // Update CLI.InVals to use outside of this function.
8180 CLI.InVals = InVals;
8182 // Verify that the target's LowerCall behaved as expected.
8183 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
8184 "LowerCall didn't return a valid chain!");
8185 assert((!CLI.IsTailCall || InVals.empty()) &&
8186 "LowerCall emitted a return value for a tail call!");
8187 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
8188 "LowerCall didn't emit the correct number of values!");
8190 // For a tail call, the return value is merely live-out and there aren't
8191 // any nodes in the DAG representing it. Return a special value to
8192 // indicate that a tail call has been emitted and no more Instructions
8193 // should be processed in the current block.
8194 if (CLI.IsTailCall) {
8195 CLI.DAG.setRoot(CLI.Chain);
8196 return std::make_pair(SDValue(), SDValue());
8200 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
8201 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
8202 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
8203 "LowerCall emitted a value with the wrong type!");
8207 SmallVector<SDValue, 4> ReturnValues;
8208 if (!CanLowerReturn) {
8209 // The instruction result is the result of loading from the
8210 // hidden sret parameter.
8211 SmallVector<EVT, 1> PVTs;
8212 Type *PtrRetTy = PointerType::getUnqual(OrigRetTy);
8214 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
8215 assert(PVTs.size() == 1 && "Pointers should fit in one register");
8216 EVT PtrVT = PVTs[0];
8218 unsigned NumValues = RetTys.size();
8219 ReturnValues.resize(NumValues);
8220 SmallVector<SDValue, 4> Chains(NumValues);
8222 // An aggregate return value cannot wrap around the address space, so
8223 // offsets to its parts don't wrap either.
8225 Flags.setNoUnsignedWrap(true);
8227 for (unsigned i = 0; i < NumValues; ++i) {
8228 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
8229 CLI.DAG.getConstant(Offsets[i], CLI.DL,
8231 SDValue L = CLI.DAG.getLoad(
8232 RetTys[i], CLI.DL, CLI.Chain, Add,
8233 MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
8234 DemoteStackIdx, Offsets[i]),
8235 /* Alignment = */ 1);
8236 ReturnValues[i] = L;
8237 Chains[i] = L.getValue(1);
8240 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
8242 // Collect the legal value parts into potentially illegal values
8243 // that correspond to the original function's return values.
8244 Optional<ISD::NodeType> AssertOp;
8246 AssertOp = ISD::AssertSext;
8247 else if (CLI.RetZExt)
8248 AssertOp = ISD::AssertZext;
8249 unsigned CurReg = 0;
8250 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8253 getRegisterTypeForCallingConv(CLI.RetTy->getContext(), VT);
8255 getNumRegistersForCallingConv(CLI.RetTy->getContext(), VT);
8257 ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
8258 NumRegs, RegisterVT, VT, nullptr,
8263 // For a function returning void, there is no return value. We can't create
8264 // such a node, so we just return a null return value in that case. In
8265 // that case, nothing will actually look at the value.
8266 if (ReturnValues.empty())
8267 return std::make_pair(SDValue(), CLI.Chain);
8270 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
8271 CLI.DAG.getVTList(RetTys), ReturnValues);
8272 return std::make_pair(Res, CLI.Chain);
8275 void TargetLowering::LowerOperationWrapper(SDNode *N,
8276 SmallVectorImpl<SDValue> &Results,
8277 SelectionDAG &DAG) const {
8278 if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
8279 Results.push_back(Res);
8282 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
8283 llvm_unreachable("LowerOperation not implemented for this target!");
8287 SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
8288 SDValue Op = getNonRegisterValue(V);
8289 assert((Op.getOpcode() != ISD::CopyFromReg ||
8290 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
8291 "Copy from a reg to the same reg!");
8292 assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg");
8294 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8295 // If this is an InlineAsm we have to match the registers required, not the
8296 // notional registers required by the type.
8298 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
8299 V->getType(), isABIRegCopy(V));
8300 SDValue Chain = DAG.getEntryNode();
8302 ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
8303 FuncInfo.PreferredExtendType.end())
8305 : FuncInfo.PreferredExtendType[V];
8306 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
8307 PendingExports.push_back(Chain);
8310 #include "llvm/CodeGen/SelectionDAGISel.h"
8312 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
8313 /// entry block, return true. This includes arguments used by switches, since
8314 /// the switch may expand into multiple basic blocks.
8315 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
8316 // With FastISel active, we may be splitting blocks, so force creation
8317 // of virtual registers for all non-dead arguments.
8319 return A->use_empty();
8321 const BasicBlock &Entry = A->getParent()->front();
8322 for (const User *U : A->users())
8323 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
8324 return false; // Use not in entry block.
8329 typedef DenseMap<const Argument *,
8330 std::pair<const AllocaInst *, const StoreInst *>>
8331 ArgCopyElisionMapTy;
8333 /// Scan the entry block of the function in FuncInfo for arguments that look
8334 /// like copies into a local alloca. Record any copied arguments in
8335 /// ArgCopyElisionCandidates.
8337 findArgumentCopyElisionCandidates(const DataLayout &DL,
8338 FunctionLoweringInfo *FuncInfo,
8339 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
8340 // Record the state of every static alloca used in the entry block. Argument
8341 // allocas are all used in the entry block, so we need approximately as many
8342 // entries as we have arguments.
8343 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
8344 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
8345 unsigned NumArgs = FuncInfo->Fn->arg_size();
8346 StaticAllocas.reserve(NumArgs * 2);
8348 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
8351 V = V->stripPointerCasts();
8352 const auto *AI = dyn_cast<AllocaInst>(V);
8353 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
8355 auto Iter = StaticAllocas.insert({AI, Unknown});
8356 return &Iter.first->second;
8359 // Look for stores of arguments to static allocas. Look through bitcasts and
8360 // GEPs to handle type coercions, as long as the alloca is fully initialized
8361 // by the store. Any non-store use of an alloca escapes it and any subsequent
8362 // unanalyzed store might write it.
8363 // FIXME: Handle structs initialized with multiple stores.
8364 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
8365 // Look for stores, and handle non-store uses conservatively.
8366 const auto *SI = dyn_cast<StoreInst>(&I);
8368 // We will look through cast uses, so ignore them completely.
8371 // Ignore debug info intrinsics, they don't escape or store to allocas.
8372 if (isa<DbgInfoIntrinsic>(I))
8374 // This is an unknown instruction. Assume it escapes or writes to all
8375 // static alloca operands.
8376 for (const Use &U : I.operands()) {
8377 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
8378 *Info = StaticAllocaInfo::Clobbered;
8383 // If the stored value is a static alloca, mark it as escaped.
8384 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
8385 *Info = StaticAllocaInfo::Clobbered;
8387 // Check if the destination is a static alloca.
8388 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
8389 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
8392 const AllocaInst *AI = cast<AllocaInst>(Dst);
8394 // Skip allocas that have been initialized or clobbered.
8395 if (*Info != StaticAllocaInfo::Unknown)
8398 // Check if the stored value is an argument, and that this store fully
8399 // initializes the alloca. Don't elide copies from the same argument twice.
8400 const Value *Val = SI->getValueOperand()->stripPointerCasts();
8401 const auto *Arg = dyn_cast<Argument>(Val);
8402 if (!Arg || Arg->hasInAllocaAttr() || Arg->hasByValAttr() ||
8403 Arg->getType()->isEmptyTy() ||
8404 DL.getTypeStoreSize(Arg->getType()) !=
8405 DL.getTypeAllocSize(AI->getAllocatedType()) ||
8406 ArgCopyElisionCandidates.count(Arg)) {
8407 *Info = StaticAllocaInfo::Clobbered;
8411 DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI << '\n');
8413 // Mark this alloca and store for argument copy elision.
8414 *Info = StaticAllocaInfo::Elidable;
8415 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
8417 // Stop scanning if we've seen all arguments. This will happen early in -O0
8418 // builds, which is useful, because -O0 builds have large entry blocks and
8420 if (ArgCopyElisionCandidates.size() == NumArgs)
8425 /// Try to elide argument copies from memory into a local alloca. Succeeds if
8426 /// ArgVal is a load from a suitable fixed stack object.
8427 static void tryToElideArgumentCopy(
8428 FunctionLoweringInfo *FuncInfo, SmallVectorImpl<SDValue> &Chains,
8429 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
8430 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
8431 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
8432 SDValue ArgVal, bool &ArgHasUses) {
8433 // Check if this is a load from a fixed stack object.
8434 auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
8437 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
8441 // Check that the fixed stack object is the right size and alignment.
8442 // Look at the alignment that the user wrote on the alloca instead of looking
8443 // at the stack object.
8444 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
8445 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
8446 const AllocaInst *AI = ArgCopyIter->second.first;
8447 int FixedIndex = FINode->getIndex();
8448 int &AllocaIndex = FuncInfo->StaticAllocaMap[AI];
8449 int OldIndex = AllocaIndex;
8450 MachineFrameInfo &MFI = FuncInfo->MF->getFrameInfo();
8451 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
8452 DEBUG(dbgs() << " argument copy elision failed due to bad fixed stack "
8456 unsigned RequiredAlignment = AI->getAlignment();
8457 if (!RequiredAlignment) {
8458 RequiredAlignment = FuncInfo->MF->getDataLayout().getABITypeAlignment(
8459 AI->getAllocatedType());
8461 if (MFI.getObjectAlignment(FixedIndex) < RequiredAlignment) {
8462 DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
8463 "greater than stack argument alignment ("
8464 << RequiredAlignment << " vs "
8465 << MFI.getObjectAlignment(FixedIndex) << ")\n");
8469 // Perform the elision. Delete the old stack object and replace its only use
8470 // in the variable info map. Mark the stack object as mutable.
8472 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
8473 << " Replacing frame index " << OldIndex << " with " << FixedIndex
8476 MFI.RemoveStackObject(OldIndex);
8477 MFI.setIsImmutableObjectIndex(FixedIndex, false);
8478 AllocaIndex = FixedIndex;
8479 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
8480 Chains.push_back(ArgVal.getValue(1));
8482 // Avoid emitting code for the store implementing the copy.
8483 const StoreInst *SI = ArgCopyIter->second.second;
8484 ElidedArgCopyInstrs.insert(SI);
8486 // Check for uses of the argument again so that we can avoid exporting ArgVal
8487 // if it is't used by anything other than the store.
8488 for (const Value *U : Arg.users()) {
8496 void SelectionDAGISel::LowerArguments(const Function &F) {
8497 SelectionDAG &DAG = SDB->DAG;
8498 SDLoc dl = SDB->getCurSDLoc();
8499 const DataLayout &DL = DAG.getDataLayout();
8500 SmallVector<ISD::InputArg, 16> Ins;
8502 if (!FuncInfo->CanLowerReturn) {
8503 // Put in an sret pointer parameter before all the other parameters.
8504 SmallVector<EVT, 1> ValueVTs;
8505 ComputeValueVTs(*TLI, DAG.getDataLayout(),
8506 PointerType::getUnqual(F.getReturnType()), ValueVTs);
8508 // NOTE: Assuming that a pointer will never break down to more than one VT
8510 ISD::ArgFlagsTy Flags;
8512 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
8513 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
8514 ISD::InputArg::NoArgIndex, 0);
8515 Ins.push_back(RetArg);
8518 // Look for stores of arguments to static allocas. Mark such arguments with a
8519 // flag to ask the target to give us the memory location of that argument if
8521 ArgCopyElisionMapTy ArgCopyElisionCandidates;
8522 findArgumentCopyElisionCandidates(DL, FuncInfo, ArgCopyElisionCandidates);
8524 // Set up the incoming argument description vector.
8525 for (const Argument &Arg : F.args()) {
8526 unsigned ArgNo = Arg.getArgNo();
8527 SmallVector<EVT, 4> ValueVTs;
8528 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
8529 bool isArgValueUsed = !Arg.use_empty();
8530 unsigned PartBase = 0;
8531 Type *FinalType = Arg.getType();
8532 if (Arg.hasAttribute(Attribute::ByVal))
8533 FinalType = cast<PointerType>(FinalType)->getElementType();
8534 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
8535 FinalType, F.getCallingConv(), F.isVarArg());
8536 for (unsigned Value = 0, NumValues = ValueVTs.size();
8537 Value != NumValues; ++Value) {
8538 EVT VT = ValueVTs[Value];
8539 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
8540 ISD::ArgFlagsTy Flags;
8542 // Certain targets (such as MIPS), may have a different ABI alignment
8543 // for a type depending on the context. Give the target a chance to
8544 // specify the alignment it wants.
8545 unsigned OriginalAlignment =
8546 TLI->getABIAlignmentForCallingConv(ArgTy, DL);
8548 if (Arg.hasAttribute(Attribute::ZExt))
8550 if (Arg.hasAttribute(Attribute::SExt))
8552 if (Arg.hasAttribute(Attribute::InReg)) {
8553 // If we are using vectorcall calling convention, a structure that is
8554 // passed InReg - is surely an HVA
8555 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
8556 isa<StructType>(Arg.getType())) {
8557 // The first value of a structure is marked
8559 Flags.setHvaStart();
8565 if (Arg.hasAttribute(Attribute::StructRet))
8567 if (Arg.hasAttribute(Attribute::SwiftSelf))
8568 Flags.setSwiftSelf();
8569 if (Arg.hasAttribute(Attribute::SwiftError))
8570 Flags.setSwiftError();
8571 if (Arg.hasAttribute(Attribute::ByVal))
8573 if (Arg.hasAttribute(Attribute::InAlloca)) {
8574 Flags.setInAlloca();
8575 // Set the byval flag for CCAssignFn callbacks that don't know about
8576 // inalloca. This way we can know how many bytes we should've allocated
8577 // and how many bytes a callee cleanup function will pop. If we port
8578 // inalloca to more targets, we'll have to add custom inalloca handling
8579 // in the various CC lowering callbacks.
8582 if (F.getCallingConv() == CallingConv::X86_INTR) {
8583 // IA Interrupt passes frame (1st parameter) by value in the stack.
8587 if (Flags.isByVal() || Flags.isInAlloca()) {
8588 PointerType *Ty = cast<PointerType>(Arg.getType());
8589 Type *ElementTy = Ty->getElementType();
8590 Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8591 // For ByVal, alignment should be passed from FE. BE will guess if
8592 // this info is not there but there are cases it cannot get right.
8593 unsigned FrameAlign;
8594 if (Arg.getParamAlignment())
8595 FrameAlign = Arg.getParamAlignment();
8597 FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
8598 Flags.setByValAlign(FrameAlign);
8600 if (Arg.hasAttribute(Attribute::Nest))
8603 Flags.setInConsecutiveRegs();
8604 Flags.setOrigAlign(OriginalAlignment);
8605 if (ArgCopyElisionCandidates.count(&Arg))
8606 Flags.setCopyElisionCandidate();
8609 TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(), VT);
8611 TLI->getNumRegistersForCallingConv(*CurDAG->getContext(), VT);
8612 for (unsigned i = 0; i != NumRegs; ++i) {
8613 ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
8614 ArgNo, PartBase+i*RegisterVT.getStoreSize());
8615 if (NumRegs > 1 && i == 0)
8616 MyFlags.Flags.setSplit();
8617 // if it isn't first piece, alignment must be 1
8619 MyFlags.Flags.setOrigAlign(1);
8620 if (i == NumRegs - 1)
8621 MyFlags.Flags.setSplitEnd();
8623 Ins.push_back(MyFlags);
8625 if (NeedsRegBlock && Value == NumValues - 1)
8626 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
8627 PartBase += VT.getStoreSize();
8631 // Call the target to set up the argument values.
8632 SmallVector<SDValue, 8> InVals;
8633 SDValue NewRoot = TLI->LowerFormalArguments(
8634 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
8636 // Verify that the target's LowerFormalArguments behaved as expected.
8637 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
8638 "LowerFormalArguments didn't return a valid chain!");
8639 assert(InVals.size() == Ins.size() &&
8640 "LowerFormalArguments didn't emit the correct number of values!");
8642 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
8643 assert(InVals[i].getNode() &&
8644 "LowerFormalArguments emitted a null value!");
8645 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
8646 "LowerFormalArguments emitted a value with the wrong type!");
8650 // Update the DAG with the new chain value resulting from argument lowering.
8651 DAG.setRoot(NewRoot);
8653 // Set up the argument values.
8655 if (!FuncInfo->CanLowerReturn) {
8656 // Create a virtual register for the sret pointer, and put in a copy
8657 // from the sret argument into it.
8658 SmallVector<EVT, 1> ValueVTs;
8659 ComputeValueVTs(*TLI, DAG.getDataLayout(),
8660 PointerType::getUnqual(F.getReturnType()), ValueVTs);
8661 MVT VT = ValueVTs[0].getSimpleVT();
8662 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
8663 Optional<ISD::NodeType> AssertOp = None;
8664 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1,
8665 RegVT, VT, nullptr, AssertOp);
8667 MachineFunction& MF = SDB->DAG.getMachineFunction();
8668 MachineRegisterInfo& RegInfo = MF.getRegInfo();
8669 unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
8670 FuncInfo->DemoteRegister = SRetReg;
8672 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
8673 DAG.setRoot(NewRoot);
8675 // i indexes lowered arguments. Bump it past the hidden sret argument.
8679 SmallVector<SDValue, 4> Chains;
8680 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
8681 for (const Argument &Arg : F.args()) {
8682 SmallVector<SDValue, 4> ArgValues;
8683 SmallVector<EVT, 4> ValueVTs;
8684 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
8685 unsigned NumValues = ValueVTs.size();
8689 bool ArgHasUses = !Arg.use_empty();
8691 // Elide the copying store if the target loaded this argument from a
8692 // suitable fixed stack object.
8693 if (Ins[i].Flags.isCopyElisionCandidate()) {
8694 tryToElideArgumentCopy(FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
8695 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
8696 InVals[i], ArgHasUses);
8699 // If this argument is unused then remember its value. It is used to generate
8700 // debugging information.
8701 bool isSwiftErrorArg =
8702 TLI->supportSwiftError() &&
8703 Arg.hasAttribute(Attribute::SwiftError);
8704 if (!ArgHasUses && !isSwiftErrorArg) {
8705 SDB->setUnusedArgValue(&Arg, InVals[i]);
8707 // Also remember any frame index for use in FastISel.
8708 if (FrameIndexSDNode *FI =
8709 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
8710 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
8713 for (unsigned Val = 0; Val != NumValues; ++Val) {
8714 EVT VT = ValueVTs[Val];
8716 TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(), VT);
8718 TLI->getNumRegistersForCallingConv(*CurDAG->getContext(), VT);
8720 // Even an apparant 'unused' swifterror argument needs to be returned. So
8721 // we do generate a copy for it that can be used on return from the
8723 if (ArgHasUses || isSwiftErrorArg) {
8724 Optional<ISD::NodeType> AssertOp;
8725 if (Arg.hasAttribute(Attribute::SExt))
8726 AssertOp = ISD::AssertSext;
8727 else if (Arg.hasAttribute(Attribute::ZExt))
8728 AssertOp = ISD::AssertZext;
8730 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
8731 PartVT, VT, nullptr, AssertOp,
8738 // We don't need to do anything else for unused arguments.
8739 if (ArgValues.empty())
8742 // Note down frame index.
8743 if (FrameIndexSDNode *FI =
8744 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
8745 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
8747 SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
8748 SDB->getCurSDLoc());
8750 SDB->setValue(&Arg, Res);
8751 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
8752 if (LoadSDNode *LNode =
8753 dyn_cast<LoadSDNode>(Res.getOperand(0).getNode()))
8754 if (FrameIndexSDNode *FI =
8755 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
8756 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
8759 // Update the SwiftErrorVRegDefMap.
8760 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
8761 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
8762 if (TargetRegisterInfo::isVirtualRegister(Reg))
8763 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB,
8764 FuncInfo->SwiftErrorArg, Reg);
8767 // If this argument is live outside of the entry block, insert a copy from
8768 // wherever we got it to the vreg that other BB's will reference it as.
8769 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
8770 // If we can, though, try to skip creating an unnecessary vreg.
8771 // FIXME: This isn't very clean... it would be nice to make this more
8772 // general. It's also subtly incompatible with the hacks FastISel
8774 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
8775 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
8776 FuncInfo->ValueMap[&Arg] = Reg;
8780 if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
8781 FuncInfo->InitializeRegForValue(&Arg);
8782 SDB->CopyToExportRegsIfNeeded(&Arg);
8786 if (!Chains.empty()) {
8787 Chains.push_back(NewRoot);
8788 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
8791 DAG.setRoot(NewRoot);
8793 assert(i == InVals.size() && "Argument register count mismatch!");
8795 // If any argument copy elisions occurred and we have debug info, update the
8796 // stale frame indices used in the dbg.declare variable info table.
8797 MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
8798 if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
8799 for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
8800 auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
8801 if (I != ArgCopyElisionFrameIndexMap.end())
8802 VI.Slot = I->second;
8806 // Finally, if the target has anything special to do, allow it to do so.
8807 EmitFunctionEntryCode();
8810 /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
8811 /// ensure constants are generated when needed. Remember the virtual registers
8812 /// that need to be added to the Machine PHI nodes as input. We cannot just
8813 /// directly add them, because expansion might result in multiple MBB's for one
8814 /// BB. As such, the start of the BB might correspond to a different MBB than
8818 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
8819 const TerminatorInst *TI = LLVMBB->getTerminator();
8821 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
8823 // Check PHI nodes in successors that expect a value to be available from this
8825 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
8826 const BasicBlock *SuccBB = TI->getSuccessor(succ);
8827 if (!isa<PHINode>(SuccBB->begin())) continue;
8828 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
8830 // If this terminator has multiple identical successors (common for
8831 // switches), only handle each succ once.
8832 if (!SuccsHandled.insert(SuccMBB).second)
8835 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
8837 // At this point we know that there is a 1-1 correspondence between LLVM PHI
8838 // nodes and Machine PHI nodes, but the incoming operands have not been
8840 for (BasicBlock::const_iterator I = SuccBB->begin();
8841 const PHINode *PN = dyn_cast<PHINode>(I); ++I) {
8842 // Ignore dead phi's.
8843 if (PN->use_empty()) continue;
8846 if (PN->getType()->isEmptyTy())
8850 const Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
8852 if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
8853 unsigned &RegOut = ConstantsOut[C];
8855 RegOut = FuncInfo.CreateRegs(C->getType());
8856 CopyValueToVirtualRegister(C, RegOut);
8860 DenseMap<const Value *, unsigned>::iterator I =
8861 FuncInfo.ValueMap.find(PHIOp);
8862 if (I != FuncInfo.ValueMap.end())
8865 assert(isa<AllocaInst>(PHIOp) &&
8866 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
8867 "Didn't codegen value into a register!??");
8868 Reg = FuncInfo.CreateRegs(PHIOp->getType());
8869 CopyValueToVirtualRegister(PHIOp, Reg);
8873 // Remember that this register needs to added to the machine PHI node as
8874 // the input for this MBB.
8875 SmallVector<EVT, 4> ValueVTs;
8876 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8877 ComputeValueVTs(TLI, DAG.getDataLayout(), PN->getType(), ValueVTs);
8878 for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
8879 EVT VT = ValueVTs[vti];
8880 unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
8881 for (unsigned i = 0, e = NumRegisters; i != e; ++i)
8882 FuncInfo.PHINodesToUpdate.push_back(
8883 std::make_pair(&*MBBI++, Reg + i));
8884 Reg += NumRegisters;
8889 ConstantsOut.clear();
8892 /// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
8895 SelectionDAGBuilder::StackProtectorDescriptor::
8896 AddSuccessorMBB(const BasicBlock *BB,
8897 MachineBasicBlock *ParentMBB,
8899 MachineBasicBlock *SuccMBB) {
8900 // If SuccBB has not been created yet, create it.
8902 MachineFunction *MF = ParentMBB->getParent();
8903 MachineFunction::iterator BBI(ParentMBB);
8904 SuccMBB = MF->CreateMachineBasicBlock(BB);
8905 MF->insert(++BBI, SuccMBB);
8907 // Add it as a successor of ParentMBB.
8908 ParentMBB->addSuccessor(
8909 SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely));
8913 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
8914 MachineFunction::iterator I(MBB);
8915 if (++I == FuncInfo.MF->end())
8920 /// During lowering new call nodes can be created (such as memset, etc.).
8921 /// Those will become new roots of the current DAG, but complications arise
8922 /// when they are tail calls. In such cases, the call lowering will update
8923 /// the root, but the builder still needs to know that a tail call has been
8924 /// lowered in order to avoid generating an additional return.
8925 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
8926 // If the node is null, we do have a tail call.
8927 if (MaybeTC.getNode() != nullptr)
8928 DAG.setRoot(MaybeTC);
8934 SelectionDAGBuilder::getJumpTableRange(const CaseClusterVector &Clusters,
8935 unsigned First, unsigned Last) const {
8936 assert(Last >= First);
8937 const APInt &LowCase = Clusters[First].Low->getValue();
8938 const APInt &HighCase = Clusters[Last].High->getValue();
8939 assert(LowCase.getBitWidth() == HighCase.getBitWidth());
8941 // FIXME: A range of consecutive cases has 100% density, but only requires one
8942 // comparison to lower. We should discriminate against such consecutive ranges
8945 return (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100) + 1;
8948 uint64_t SelectionDAGBuilder::getJumpTableNumCases(
8949 const SmallVectorImpl<unsigned> &TotalCases, unsigned First,
8950 unsigned Last) const {
8951 assert(Last >= First);
8952 assert(TotalCases[Last] >= TotalCases[First]);
8954 TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
8958 bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters,
8959 unsigned First, unsigned Last,
8960 const SwitchInst *SI,
8961 MachineBasicBlock *DefaultMBB,
8962 CaseCluster &JTCluster) {
8963 assert(First <= Last);
8965 auto Prob = BranchProbability::getZero();
8966 unsigned NumCmps = 0;
8967 std::vector<MachineBasicBlock*> Table;
8968 DenseMap<MachineBasicBlock*, BranchProbability> JTProbs;
8970 // Initialize probabilities in JTProbs.
8971 for (unsigned I = First; I <= Last; ++I)
8972 JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
8974 for (unsigned I = First; I <= Last; ++I) {
8975 assert(Clusters[I].Kind == CC_Range);
8976 Prob += Clusters[I].Prob;
8977 const APInt &Low = Clusters[I].Low->getValue();
8978 const APInt &High = Clusters[I].High->getValue();
8979 NumCmps += (Low == High) ? 1 : 2;
8981 // Fill the gap between this and the previous cluster.
8982 const APInt &PreviousHigh = Clusters[I - 1].High->getValue();
8983 assert(PreviousHigh.slt(Low));
8984 uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
8985 for (uint64_t J = 0; J < Gap; J++)
8986 Table.push_back(DefaultMBB);
8988 uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
8989 for (uint64_t J = 0; J < ClusterSize; ++J)
8990 Table.push_back(Clusters[I].MBB);
8991 JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
8994 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8995 unsigned NumDests = JTProbs.size();
8996 if (TLI.isSuitableForBitTests(
8997 NumDests, NumCmps, Clusters[First].Low->getValue(),
8998 Clusters[Last].High->getValue(), DAG.getDataLayout())) {
8999 // Clusters[First..Last] should be lowered as bit tests instead.
9003 // Create the MBB that will load from and jump through the table.
9004 // Note: We create it here, but it's not inserted into the function yet.
9005 MachineFunction *CurMF = FuncInfo.MF;
9006 MachineBasicBlock *JumpTableMBB =
9007 CurMF->CreateMachineBasicBlock(SI->getParent());
9009 // Add successors. Note: use table order for determinism.
9010 SmallPtrSet<MachineBasicBlock *, 8> Done;
9011 for (MachineBasicBlock *Succ : Table) {
9012 if (Done.count(Succ))
9014 addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
9017 JumpTableMBB->normalizeSuccProbs();
9019 unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
9020 ->createJumpTableIndex(Table);
9022 // Set up the jump table info.
9023 JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
9024 JumpTableHeader JTH(Clusters[First].Low->getValue(),
9025 Clusters[Last].High->getValue(), SI->getCondition(),
9027 JTCases.emplace_back(std::move(JTH), std::move(JT));
9029 JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
9030 JTCases.size() - 1, Prob);
9034 void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
9035 const SwitchInst *SI,
9036 MachineBasicBlock *DefaultMBB) {
9038 // Clusters must be non-empty, sorted, and only contain Range clusters.
9039 assert(!Clusters.empty());
9040 for (CaseCluster &C : Clusters)
9041 assert(C.Kind == CC_Range);
9042 for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
9043 assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()));
9046 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9047 if (!TLI.areJTsAllowed(SI->getParent()->getParent()))
9050 const int64_t N = Clusters.size();
9051 const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries();
9052 const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2;
9054 if (N < 2 || N < MinJumpTableEntries)
9057 // TotalCases[i]: Total nbr of cases in Clusters[0..i].
9058 SmallVector<unsigned, 8> TotalCases(N);
9059 for (unsigned i = 0; i < N; ++i) {
9060 const APInt &Hi = Clusters[i].High->getValue();
9061 const APInt &Lo = Clusters[i].Low->getValue();
9062 TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
9064 TotalCases[i] += TotalCases[i - 1];
9067 // Cheap case: the whole range may be suitable for jump table.
9068 uint64_t Range = getJumpTableRange(Clusters,0, N - 1);
9069 uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1);
9070 assert(NumCases < UINT64_MAX / 100);
9071 assert(Range >= NumCases);
9072 if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9073 CaseCluster JTCluster;
9074 if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
9075 Clusters[0] = JTCluster;
9081 // The algorithm below is not suitable for -O0.
9082 if (TM.getOptLevel() == CodeGenOpt::None)
9085 // Split Clusters into minimum number of dense partitions. The algorithm uses
9086 // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
9087 // for the Case Statement'" (1994), but builds the MinPartitions array in
9088 // reverse order to make it easier to reconstruct the partitions in ascending
9089 // order. In the choice between two optimal partitionings, it picks the one
9090 // which yields more jump tables.
9092 // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9093 SmallVector<unsigned, 8> MinPartitions(N);
9094 // LastElement[i] is the last element of the partition starting at i.
9095 SmallVector<unsigned, 8> LastElement(N);
9096 // PartitionsScore[i] is used to break ties when choosing between two
9097 // partitionings resulting in the same number of partitions.
9098 SmallVector<unsigned, 8> PartitionsScore(N);
9099 // For PartitionsScore, a small number of comparisons is considered as good as
9100 // a jump table and a single comparison is considered better than a jump
9102 enum PartitionScores : unsigned {
9109 // Base case: There is only one way to partition Clusters[N-1].
9110 MinPartitions[N - 1] = 1;
9111 LastElement[N - 1] = N - 1;
9112 PartitionsScore[N - 1] = PartitionScores::SingleCase;
9114 // Note: loop indexes are signed to avoid underflow.
9115 for (int64_t i = N - 2; i >= 0; i--) {
9116 // Find optimal partitioning of Clusters[i..N-1].
9117 // Baseline: Put Clusters[i] into a partition on its own.
9118 MinPartitions[i] = MinPartitions[i + 1] + 1;
9120 PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase;
9122 // Search for a solution that results in fewer partitions.
9123 for (int64_t j = N - 1; j > i; j--) {
9124 // Try building a partition from Clusters[i..j].
9125 uint64_t Range = getJumpTableRange(Clusters, i, j);
9126 uint64_t NumCases = getJumpTableNumCases(TotalCases, i, j);
9127 assert(NumCases < UINT64_MAX / 100);
9128 assert(Range >= NumCases);
9129 if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9130 unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9131 unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1];
9132 int64_t NumEntries = j - i + 1;
9134 if (NumEntries == 1)
9135 Score += PartitionScores::SingleCase;
9136 else if (NumEntries <= SmallNumberOfEntries)
9137 Score += PartitionScores::FewCases;
9138 else if (NumEntries >= MinJumpTableEntries)
9139 Score += PartitionScores::Table;
9141 // If this leads to fewer partitions, or to the same number of
9142 // partitions with better score, it is a better partitioning.
9143 if (NumPartitions < MinPartitions[i] ||
9144 (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) {
9145 MinPartitions[i] = NumPartitions;
9147 PartitionsScore[i] = Score;
9153 // Iterate over the partitions, replacing some with jump tables in-place.
9154 unsigned DstIndex = 0;
9155 for (unsigned First = 0, Last; First < N; First = Last + 1) {
9156 Last = LastElement[First];
9157 assert(Last >= First);
9158 assert(DstIndex <= First);
9159 unsigned NumClusters = Last - First + 1;
9161 CaseCluster JTCluster;
9162 if (NumClusters >= MinJumpTableEntries &&
9163 buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
9164 Clusters[DstIndex++] = JTCluster;
9166 for (unsigned I = First; I <= Last; ++I)
9167 std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
9170 Clusters.resize(DstIndex);
9173 bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
9174 unsigned First, unsigned Last,
9175 const SwitchInst *SI,
9176 CaseCluster &BTCluster) {
9177 assert(First <= Last);
9181 BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9182 unsigned NumCmps = 0;
9183 for (int64_t I = First; I <= Last; ++I) {
9184 assert(Clusters[I].Kind == CC_Range);
9185 Dests.set(Clusters[I].MBB->getNumber());
9186 NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
9188 unsigned NumDests = Dests.count();
9190 APInt Low = Clusters[First].Low->getValue();
9191 APInt High = Clusters[Last].High->getValue();
9192 assert(Low.slt(High));
9194 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9195 const DataLayout &DL = DAG.getDataLayout();
9196 if (!TLI.isSuitableForBitTests(NumDests, NumCmps, Low, High, DL))
9202 const int BitWidth = TLI.getPointerTy(DL).getSizeInBits();
9203 assert(TLI.rangeFitsInWord(Low, High, DL) &&
9204 "Case range must fit in bit mask!");
9206 // Check if the clusters cover a contiguous range such that no value in the
9207 // range will jump to the default statement.
9208 bool ContiguousRange = true;
9209 for (int64_t I = First + 1; I <= Last; ++I) {
9210 if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) {
9211 ContiguousRange = false;
9216 if (Low.isStrictlyPositive() && High.slt(BitWidth)) {
9217 // Optimize the case where all the case values fit in a word without having
9218 // to subtract minValue. In this case, we can optimize away the subtraction.
9219 LowBound = APInt::getNullValue(Low.getBitWidth());
9221 ContiguousRange = false;
9224 CmpRange = High - Low;
9228 auto TotalProb = BranchProbability::getZero();
9229 for (unsigned i = First; i <= Last; ++i) {
9230 // Find the CaseBits for this destination.
9232 for (j = 0; j < CBV.size(); ++j)
9233 if (CBV[j].BB == Clusters[i].MBB)
9235 if (j == CBV.size())
9237 CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
9238 CaseBits *CB = &CBV[j];
9240 // Update Mask, Bits and ExtraProb.
9241 uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
9242 uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
9243 assert(Hi >= Lo && Hi < 64 && "Invalid bit case!");
9244 CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
9245 CB->Bits += Hi - Lo + 1;
9246 CB->ExtraProb += Clusters[i].Prob;
9247 TotalProb += Clusters[i].Prob;
9251 std::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
9252 // Sort by probability first, number of bits second.
9253 if (a.ExtraProb != b.ExtraProb)
9254 return a.ExtraProb > b.ExtraProb;
9255 return a.Bits > b.Bits;
9258 for (auto &CB : CBV) {
9259 MachineBasicBlock *BitTestBB =
9260 FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
9261 BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
9263 BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
9264 SI->getCondition(), -1U, MVT::Other, false,
9265 ContiguousRange, nullptr, nullptr, std::move(BTI),
9268 BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
9269 BitTestCases.size() - 1, TotalProb);
9273 void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
9274 const SwitchInst *SI) {
9275 // Partition Clusters into as few subsets as possible, where each subset has a
9276 // range that fits in a machine word and has <= 3 unique destinations.
9279 // Clusters must be sorted and contain Range or JumpTable clusters.
9280 assert(!Clusters.empty());
9281 assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable);
9282 for (const CaseCluster &C : Clusters)
9283 assert(C.Kind == CC_Range || C.Kind == CC_JumpTable);
9284 for (unsigned i = 1; i < Clusters.size(); ++i)
9285 assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()));
9288 // The algorithm below is not suitable for -O0.
9289 if (TM.getOptLevel() == CodeGenOpt::None)
9292 // If target does not have legal shift left, do not emit bit tests at all.
9293 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9294 const DataLayout &DL = DAG.getDataLayout();
9296 EVT PTy = TLI.getPointerTy(DL);
9297 if (!TLI.isOperationLegal(ISD::SHL, PTy))
9300 int BitWidth = PTy.getSizeInBits();
9301 const int64_t N = Clusters.size();
9303 // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9304 SmallVector<unsigned, 8> MinPartitions(N);
9305 // LastElement[i] is the last element of the partition starting at i.
9306 SmallVector<unsigned, 8> LastElement(N);
9308 // FIXME: This might not be the best algorithm for finding bit test clusters.
9310 // Base case: There is only one way to partition Clusters[N-1].
9311 MinPartitions[N - 1] = 1;
9312 LastElement[N - 1] = N - 1;
9314 // Note: loop indexes are signed to avoid underflow.
9315 for (int64_t i = N - 2; i >= 0; --i) {
9316 // Find optimal partitioning of Clusters[i..N-1].
9317 // Baseline: Put Clusters[i] into a partition on its own.
9318 MinPartitions[i] = MinPartitions[i + 1] + 1;
9321 // Search for a solution that results in fewer partitions.
9322 // Note: the search is limited by BitWidth, reducing time complexity.
9323 for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
9324 // Try building a partition from Clusters[i..j].
9327 if (!TLI.rangeFitsInWord(Clusters[i].Low->getValue(),
9328 Clusters[j].High->getValue(), DL))
9331 // Check nbr of destinations and cluster types.
9332 // FIXME: This works, but doesn't seem very efficient.
9333 bool RangesOnly = true;
9334 BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9335 for (int64_t k = i; k <= j; k++) {
9336 if (Clusters[k].Kind != CC_Range) {
9340 Dests.set(Clusters[k].MBB->getNumber());
9342 if (!RangesOnly || Dests.count() > 3)
9345 // Check if it's a better partition.
9346 unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9347 if (NumPartitions < MinPartitions[i]) {
9348 // Found a better partition.
9349 MinPartitions[i] = NumPartitions;
9355 // Iterate over the partitions, replacing with bit-test clusters in-place.
9356 unsigned DstIndex = 0;
9357 for (unsigned First = 0, Last; First < N; First = Last + 1) {
9358 Last = LastElement[First];
9359 assert(First <= Last);
9360 assert(DstIndex <= First);
9362 CaseCluster BitTestCluster;
9363 if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
9364 Clusters[DstIndex++] = BitTestCluster;
9366 size_t NumClusters = Last - First + 1;
9367 std::memmove(&Clusters[DstIndex], &Clusters[First],
9368 sizeof(Clusters[0]) * NumClusters);
9369 DstIndex += NumClusters;
9372 Clusters.resize(DstIndex);
9375 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
9376 MachineBasicBlock *SwitchMBB,
9377 MachineBasicBlock *DefaultMBB) {
9378 MachineFunction *CurMF = FuncInfo.MF;
9379 MachineBasicBlock *NextMBB = nullptr;
9380 MachineFunction::iterator BBI(W.MBB);
9381 if (++BBI != FuncInfo.MF->end())
9384 unsigned Size = W.LastCluster - W.FirstCluster + 1;
9386 BranchProbabilityInfo *BPI = FuncInfo.BPI;
9388 if (Size == 2 && W.MBB == SwitchMBB) {
9389 // If any two of the cases has the same destination, and if one value
9390 // is the same as the other, but has one bit unset that the other has set,
9391 // use bit manipulation to do two compares at once. For example:
9392 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
9393 // TODO: This could be extended to merge any 2 cases in switches with 3
9395 // TODO: Handle cases where W.CaseBB != SwitchBB.
9396 CaseCluster &Small = *W.FirstCluster;
9397 CaseCluster &Big = *W.LastCluster;
9399 if (Small.Low == Small.High && Big.Low == Big.High &&
9400 Small.MBB == Big.MBB) {
9401 const APInt &SmallValue = Small.Low->getValue();
9402 const APInt &BigValue = Big.Low->getValue();
9404 // Check that there is only one bit different.
9405 APInt CommonBit = BigValue ^ SmallValue;
9406 if (CommonBit.isPowerOf2()) {
9407 SDValue CondLHS = getValue(Cond);
9408 EVT VT = CondLHS.getValueType();
9409 SDLoc DL = getCurSDLoc();
9411 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
9412 DAG.getConstant(CommonBit, DL, VT));
9413 SDValue Cond = DAG.getSetCC(
9414 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
9417 // Update successor info.
9418 // Both Small and Big will jump to Small.BB, so we sum up the
9420 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
9422 addSuccessorWithProb(
9423 SwitchMBB, DefaultMBB,
9424 // The default destination is the first successor in IR.
9425 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
9427 addSuccessorWithProb(SwitchMBB, DefaultMBB);
9429 // Insert the true branch.
9431 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
9432 DAG.getBasicBlock(Small.MBB));
9433 // Insert the false branch.
9434 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
9435 DAG.getBasicBlock(DefaultMBB));
9437 DAG.setRoot(BrCond);
9443 if (TM.getOptLevel() != CodeGenOpt::None) {
9444 // Order cases by probability so the most likely case will be checked first.
9445 std::sort(W.FirstCluster, W.LastCluster + 1,
9446 [](const CaseCluster &a, const CaseCluster &b) {
9447 return a.Prob > b.Prob;
9450 // Rearrange the case blocks so that the last one falls through if possible
9451 // without without changing the order of probabilities.
9452 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
9454 if (I->Prob > W.LastCluster->Prob)
9456 if (I->Kind == CC_Range && I->MBB == NextMBB) {
9457 std::swap(*I, *W.LastCluster);
9463 // Compute total probability.
9464 BranchProbability DefaultProb = W.DefaultProb;
9465 BranchProbability UnhandledProbs = DefaultProb;
9466 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
9467 UnhandledProbs += I->Prob;
9469 MachineBasicBlock *CurMBB = W.MBB;
9470 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
9471 MachineBasicBlock *Fallthrough;
9472 if (I == W.LastCluster) {
9473 // For the last cluster, fall through to the default destination.
9474 Fallthrough = DefaultMBB;
9476 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
9477 CurMF->insert(BBI, Fallthrough);
9478 // Put Cond in a virtual register to make it available from the new blocks.
9479 ExportFromCurrentBlock(Cond);
9481 UnhandledProbs -= I->Prob;
9484 case CC_JumpTable: {
9485 // FIXME: Optimize away range check based on pivot comparisons.
9486 JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
9487 JumpTable *JT = &JTCases[I->JTCasesIndex].second;
9489 // The jump block hasn't been inserted yet; insert it here.
9490 MachineBasicBlock *JumpMBB = JT->MBB;
9491 CurMF->insert(BBI, JumpMBB);
9493 auto JumpProb = I->Prob;
9494 auto FallthroughProb = UnhandledProbs;
9496 // If the default statement is a target of the jump table, we evenly
9497 // distribute the default probability to successors of CurMBB. Also
9498 // update the probability on the edge from JumpMBB to Fallthrough.
9499 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
9500 SE = JumpMBB->succ_end();
9502 if (*SI == DefaultMBB) {
9503 JumpProb += DefaultProb / 2;
9504 FallthroughProb -= DefaultProb / 2;
9505 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
9506 JumpMBB->normalizeSuccProbs();
9511 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
9512 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
9513 CurMBB->normalizeSuccProbs();
9515 // The jump table header will be inserted in our current block, do the
9516 // range check, and fall through to our fallthrough block.
9517 JTH->HeaderBB = CurMBB;
9518 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
9520 // If we're in the right place, emit the jump table header right now.
9521 if (CurMBB == SwitchMBB) {
9522 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
9523 JTH->Emitted = true;
9528 // FIXME: Optimize away range check based on pivot comparisons.
9529 BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
9531 // The bit test blocks haven't been inserted yet; insert them here.
9532 for (BitTestCase &BTC : BTB->Cases)
9533 CurMF->insert(BBI, BTC.ThisBB);
9535 // Fill in fields of the BitTestBlock.
9536 BTB->Parent = CurMBB;
9537 BTB->Default = Fallthrough;
9539 BTB->DefaultProb = UnhandledProbs;
9540 // If the cases in bit test don't form a contiguous range, we evenly
9541 // distribute the probability on the edge to Fallthrough to two
9542 // successors of CurMBB.
9543 if (!BTB->ContiguousRange) {
9544 BTB->Prob += DefaultProb / 2;
9545 BTB->DefaultProb -= DefaultProb / 2;
9548 // If we're in the right place, emit the bit test header right now.
9549 if (CurMBB == SwitchMBB) {
9550 visitBitTestHeader(*BTB, SwitchMBB);
9551 BTB->Emitted = true;
9556 const Value *RHS, *LHS, *MHS;
9558 if (I->Low == I->High) {
9559 // Check Cond == I->Low.
9565 // Check I->Low <= Cond <= I->High.
9572 // The false probability is the sum of all unhandled cases.
9573 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, I->Prob,
9576 if (CurMBB == SwitchMBB)
9577 visitSwitchCase(CB, SwitchMBB);
9579 SwitchCases.push_back(CB);
9584 CurMBB = Fallthrough;
9588 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
9589 CaseClusterIt First,
9590 CaseClusterIt Last) {
9591 return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
9592 if (X.Prob != CC.Prob)
9593 return X.Prob > CC.Prob;
9595 // Ties are broken by comparing the case value.
9596 return X.Low->getValue().slt(CC.Low->getValue());
9600 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
9601 const SwitchWorkListItem &W,
9603 MachineBasicBlock *SwitchMBB) {
9604 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
9605 "Clusters not sorted?");
9607 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
9609 // Balance the tree based on branch probabilities to create a near-optimal (in
9610 // terms of search time given key frequency) binary search tree. See e.g. Kurt
9611 // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
9612 CaseClusterIt LastLeft = W.FirstCluster;
9613 CaseClusterIt FirstRight = W.LastCluster;
9614 auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
9615 auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
9617 // Move LastLeft and FirstRight towards each other from opposite directions to
9618 // find a partitioning of the clusters which balances the probability on both
9619 // sides. If LeftProb and RightProb are equal, alternate which side is
9620 // taken to ensure 0-probability nodes are distributed evenly.
9622 while (LastLeft + 1 < FirstRight) {
9623 if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
9624 LeftProb += (++LastLeft)->Prob;
9626 RightProb += (--FirstRight)->Prob;
9631 // Our binary search tree differs from a typical BST in that ours can have up
9632 // to three values in each leaf. The pivot selection above doesn't take that
9633 // into account, which means the tree might require more nodes and be less
9634 // efficient. We compensate for this here.
9636 unsigned NumLeft = LastLeft - W.FirstCluster + 1;
9637 unsigned NumRight = W.LastCluster - FirstRight + 1;
9639 if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
9640 // If one side has less than 3 clusters, and the other has more than 3,
9641 // consider taking a cluster from the other side.
9643 if (NumLeft < NumRight) {
9644 // Consider moving the first cluster on the right to the left side.
9645 CaseCluster &CC = *FirstRight;
9646 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
9647 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
9648 if (LeftSideRank <= RightSideRank) {
9649 // Moving the cluster to the left does not demote it.
9655 assert(NumRight < NumLeft);
9656 // Consider moving the last element on the left to the right side.
9657 CaseCluster &CC = *LastLeft;
9658 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
9659 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
9660 if (RightSideRank <= LeftSideRank) {
9661 // Moving the cluster to the right does not demot it.
9671 assert(LastLeft + 1 == FirstRight);
9672 assert(LastLeft >= W.FirstCluster);
9673 assert(FirstRight <= W.LastCluster);
9675 // Use the first element on the right as pivot since we will make less-than
9676 // comparisons against it.
9677 CaseClusterIt PivotCluster = FirstRight;
9678 assert(PivotCluster > W.FirstCluster);
9679 assert(PivotCluster <= W.LastCluster);
9681 CaseClusterIt FirstLeft = W.FirstCluster;
9682 CaseClusterIt LastRight = W.LastCluster;
9684 const ConstantInt *Pivot = PivotCluster->Low;
9686 // New blocks will be inserted immediately after the current one.
9687 MachineFunction::iterator BBI(W.MBB);
9690 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
9691 // we can branch to its destination directly if it's squeezed exactly in
9692 // between the known lower bound and Pivot - 1.
9693 MachineBasicBlock *LeftMBB;
9694 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
9695 FirstLeft->Low == W.GE &&
9696 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
9697 LeftMBB = FirstLeft->MBB;
9699 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
9700 FuncInfo.MF->insert(BBI, LeftMBB);
9702 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
9703 // Put Cond in a virtual register to make it available from the new blocks.
9704 ExportFromCurrentBlock(Cond);
9707 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
9708 // single cluster, RHS.Low == Pivot, and we can branch to its destination
9709 // directly if RHS.High equals the current upper bound.
9710 MachineBasicBlock *RightMBB;
9711 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
9712 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
9713 RightMBB = FirstRight->MBB;
9715 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
9716 FuncInfo.MF->insert(BBI, RightMBB);
9718 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
9719 // Put Cond in a virtual register to make it available from the new blocks.
9720 ExportFromCurrentBlock(Cond);
9723 // Create the CaseBlock record that will be used to lower the branch.
9724 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
9725 LeftProb, RightProb);
9727 if (W.MBB == SwitchMBB)
9728 visitSwitchCase(CB, SwitchMBB);
9730 SwitchCases.push_back(CB);
9733 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
9734 // Extract cases from the switch.
9735 BranchProbabilityInfo *BPI = FuncInfo.BPI;
9736 CaseClusterVector Clusters;
9737 Clusters.reserve(SI.getNumCases());
9738 for (auto I : SI.cases()) {
9739 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
9740 const ConstantInt *CaseVal = I.getCaseValue();
9741 BranchProbability Prob =
9742 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
9743 : BranchProbability(1, SI.getNumCases() + 1);
9744 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
9747 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
9749 // Cluster adjacent cases with the same destination. We do this at all
9750 // optimization levels because it's cheap to do and will make codegen faster
9751 // if there are many clusters.
9752 sortAndRangeify(Clusters);
9754 if (TM.getOptLevel() != CodeGenOpt::None) {
9755 // Replace an unreachable default with the most popular destination.
9756 // FIXME: Exploit unreachable default more aggressively.
9757 bool UnreachableDefault =
9758 isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
9759 if (UnreachableDefault && !Clusters.empty()) {
9760 DenseMap<const BasicBlock *, unsigned> Popularity;
9761 unsigned MaxPop = 0;
9762 const BasicBlock *MaxBB = nullptr;
9763 for (auto I : SI.cases()) {
9764 const BasicBlock *BB = I.getCaseSuccessor();
9765 if (++Popularity[BB] > MaxPop) {
9766 MaxPop = Popularity[BB];
9771 assert(MaxPop > 0 && MaxBB);
9772 DefaultMBB = FuncInfo.MBBMap[MaxBB];
9774 // Remove cases that were pointing to the destination that is now the
9776 CaseClusterVector New;
9777 New.reserve(Clusters.size());
9778 for (CaseCluster &CC : Clusters) {
9779 if (CC.MBB != DefaultMBB)
9782 Clusters = std::move(New);
9786 // If there is only the default destination, jump there directly.
9787 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
9788 if (Clusters.empty()) {
9789 SwitchMBB->addSuccessor(DefaultMBB);
9790 if (DefaultMBB != NextBlock(SwitchMBB)) {
9791 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
9792 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
9797 findJumpTables(Clusters, &SI, DefaultMBB);
9798 findBitTestClusters(Clusters, &SI);
9801 dbgs() << "Case clusters: ";
9802 for (const CaseCluster &C : Clusters) {
9803 if (C.Kind == CC_JumpTable) dbgs() << "JT:";
9804 if (C.Kind == CC_BitTests) dbgs() << "BT:";
9806 C.Low->getValue().print(dbgs(), true);
9807 if (C.Low != C.High) {
9809 C.High->getValue().print(dbgs(), true);
9816 assert(!Clusters.empty());
9817 SwitchWorkList WorkList;
9818 CaseClusterIt First = Clusters.begin();
9819 CaseClusterIt Last = Clusters.end() - 1;
9820 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
9821 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
9823 while (!WorkList.empty()) {
9824 SwitchWorkListItem W = WorkList.back();
9825 WorkList.pop_back();
9826 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
9828 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
9829 !DefaultMBB->getParent()->getFunction()->optForMinSize()) {
9830 // For optimized builds, lower large range as a balanced binary tree.
9831 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
9835 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);