1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
19 //===----------------------------------------------------------------------===//
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/KnownBits.h"
25 #include "llvm/Support/raw_ostream.h"
28 #define DEBUG_TYPE "legalize-types"
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
40 SDValue Res = SDValue();
42 // See if the target wants to custom expand this node.
43 if (CustomLowerNode(N, N->getValueType(ResNo), true))
46 switch (N->getOpcode()) {
49 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
50 N->dump(&DAG); dbgs() << "\n";
52 llvm_unreachable("Do not know how to promote this operator!");
53 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
54 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
55 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
56 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
57 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
58 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
59 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
60 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
61 case ISD::CTLZ_ZERO_UNDEF:
62 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
63 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
64 case ISD::CTTZ_ZERO_UNDEF:
65 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
66 case ISD::EXTRACT_VECTOR_ELT:
67 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
68 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
69 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
71 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
73 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
74 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
75 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
76 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
78 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
80 case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
82 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
83 case ISD::SIGN_EXTEND_INREG:
84 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
85 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
86 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
87 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
88 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
89 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
91 case ISD::EXTRACT_SUBVECTOR:
92 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
93 case ISD::VECTOR_SHUFFLE:
94 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
95 case ISD::INSERT_VECTOR_ELT:
96 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
97 case ISD::BUILD_VECTOR:
98 Res = PromoteIntRes_BUILD_VECTOR(N); break;
99 case ISD::SCALAR_TO_VECTOR:
100 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
101 case ISD::CONCAT_VECTORS:
102 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
104 case ISD::ANY_EXTEND_VECTOR_INREG:
105 case ISD::SIGN_EXTEND_VECTOR_INREG:
106 case ISD::ZERO_EXTEND_VECTOR_INREG:
107 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
109 case ISD::SIGN_EXTEND:
110 case ISD::ZERO_EXTEND:
111 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
113 case ISD::FP_TO_SINT:
114 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
116 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
123 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
126 case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
129 case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
132 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
134 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
136 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
139 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
141 case ISD::ATOMIC_LOAD:
142 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
144 case ISD::ATOMIC_LOAD_ADD:
145 case ISD::ATOMIC_LOAD_SUB:
146 case ISD::ATOMIC_LOAD_AND:
147 case ISD::ATOMIC_LOAD_OR:
148 case ISD::ATOMIC_LOAD_XOR:
149 case ISD::ATOMIC_LOAD_NAND:
150 case ISD::ATOMIC_LOAD_MIN:
151 case ISD::ATOMIC_LOAD_MAX:
152 case ISD::ATOMIC_LOAD_UMIN:
153 case ISD::ATOMIC_LOAD_UMAX:
154 case ISD::ATOMIC_SWAP:
155 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
157 case ISD::ATOMIC_CMP_SWAP:
158 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
159 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
163 // If the result is null then the sub-method took care of registering it.
165 SetPromotedInteger(SDValue(N, ResNo), Res);
168 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
170 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
171 return GetPromotedInteger(Op);
174 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
175 // Sign-extend the new bits, and continue the assertion.
176 SDValue Op = SExtPromotedInteger(N->getOperand(0));
177 return DAG.getNode(ISD::AssertSext, SDLoc(N),
178 Op.getValueType(), Op, N->getOperand(1));
181 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
182 // Zero the new bits, and continue the assertion.
183 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
184 return DAG.getNode(ISD::AssertZext, SDLoc(N),
185 Op.getValueType(), Op, N->getOperand(1));
188 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
189 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
190 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
191 N->getMemoryVT(), ResVT,
192 N->getChain(), N->getBasePtr(),
194 // Legalize the chain result - switch anything that used the old chain to
196 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
200 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
201 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
202 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
204 N->getChain(), N->getBasePtr(),
205 Op2, N->getMemOperand());
206 // Legalize the chain result - switch anything that used the old chain to
208 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
212 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
215 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
216 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
217 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
219 // Only use the result of getSetCCResultType if it is legal,
220 // otherwise just use the promoted result type (NVT).
221 if (!TLI.isTypeLegal(SVT))
224 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
225 SDValue Res = DAG.getAtomicCmpSwap(
226 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
227 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
229 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
230 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
231 return Res.getValue(1);
234 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
235 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
237 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
238 SDValue Res = DAG.getAtomicCmpSwap(
239 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
240 N->getBasePtr(), Op2, Op3, N->getMemOperand());
241 // Update the use to N with the newly created Res.
242 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
243 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
247 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
248 SDValue InOp = N->getOperand(0);
249 EVT InVT = InOp.getValueType();
250 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
251 EVT OutVT = N->getValueType(0);
252 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
255 switch (getTypeAction(InVT)) {
256 case TargetLowering::TypeLegal:
258 case TargetLowering::TypePromoteInteger:
259 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
260 // The input promotes to the same size. Convert the promoted value.
261 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
263 case TargetLowering::TypeSoftenFloat:
264 // Promote the integer operand by hand.
265 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
266 case TargetLowering::TypePromoteFloat: {
267 // Convert the promoted float by hand.
268 SDValue PromotedOp = GetPromotedFloat(InOp);
269 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
272 case TargetLowering::TypeExpandInteger:
273 case TargetLowering::TypeExpandFloat:
275 case TargetLowering::TypeScalarizeVector:
276 // Convert the element to an integer and promote it by hand.
277 if (!NOutVT.isVector())
278 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
279 BitConvertToInteger(GetScalarizedVector(InOp)));
281 case TargetLowering::TypeSplitVector: {
282 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
283 // pieces of the input into integers and reassemble in the final type.
285 GetSplitVector(N->getOperand(0), Lo, Hi);
286 Lo = BitConvertToInteger(Lo);
287 Hi = BitConvertToInteger(Hi);
289 if (DAG.getDataLayout().isBigEndian())
292 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
293 EVT::getIntegerVT(*DAG.getContext(),
294 NOutVT.getSizeInBits()),
295 JoinIntegers(Lo, Hi));
296 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
298 case TargetLowering::TypeWidenVector:
299 // The input is widened to the same size. Convert to the widened value.
300 // Make sure that the outgoing value is not a vector, because this would
301 // make us bitcast between two vectors which are legalized in different ways.
302 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
303 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
306 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
307 CreateStackStoreLoad(InOp, OutVT));
310 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
311 SDValue Op = GetPromotedInteger(N->getOperand(0));
312 EVT OVT = N->getValueType(0);
313 EVT NVT = Op.getValueType();
316 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
318 ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
319 DAG.getConstant(DiffBits, dl,
320 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
323 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
324 SDValue Op = GetPromotedInteger(N->getOperand(0));
325 EVT OVT = N->getValueType(0);
326 EVT NVT = Op.getValueType();
329 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
331 ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
332 DAG.getConstant(DiffBits, dl,
333 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
336 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
337 // The pair element type may be legal, or may not promote to the same type as
338 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
339 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
340 TLI.getTypeToTransformTo(*DAG.getContext(),
341 N->getValueType(0)), JoinIntegers(N->getOperand(0),
345 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
346 EVT VT = N->getValueType(0);
347 // FIXME there is no actual debug info here
349 // Zero extend things like i1, sign extend everything else. It shouldn't
350 // matter in theory which one we pick, but this tends to give better code?
351 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
352 SDValue Result = DAG.getNode(Opc, dl,
353 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
355 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
359 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
360 // Zero extend to the promoted type and do the count there.
361 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
363 EVT OVT = N->getValueType(0);
364 EVT NVT = Op.getValueType();
365 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
366 // Subtract off the extra leading bits in the bigger type.
368 ISD::SUB, dl, NVT, Op,
369 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
373 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
374 // Zero extend to the promoted type and do the count there.
375 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
376 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
379 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
380 SDValue Op = GetPromotedInteger(N->getOperand(0));
381 EVT OVT = N->getValueType(0);
382 EVT NVT = Op.getValueType();
384 if (N->getOpcode() == ISD::CTTZ) {
385 // The count is the same in the promoted type except if the original
386 // value was zero. This can be handled by setting the bit just off
387 // the top of the original type.
388 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
389 OVT.getScalarSizeInBits());
390 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
392 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
395 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
397 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
398 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
402 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
403 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
404 unsigned NewOpc = N->getOpcode();
407 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
408 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
409 // and SINT conversions are Custom, there is no way to tell which is
410 // preferable. We choose SINT because that's the right thing on PPC.)
411 if (N->getOpcode() == ISD::FP_TO_UINT &&
412 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
413 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
414 NewOpc = ISD::FP_TO_SINT;
416 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
418 // Assert that the converted value fits in the original type. If it doesn't
419 // (eg: because the value being converted is too big), then the result of the
420 // original operation was undefined anyway, so the assert is still correct.
422 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
423 // before legalization: fp-to-uint16, 65534. -> 0xfffe
424 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
425 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
426 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
427 DAG.getValueType(N->getValueType(0).getScalarType()));
430 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
431 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
434 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
437 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
438 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
441 if (getTypeAction(N->getOperand(0).getValueType())
442 == TargetLowering::TypePromoteInteger) {
443 SDValue Res = GetPromotedInteger(N->getOperand(0));
444 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
446 // If the result and operand types are the same after promotion, simplify
447 // to an in-register extension.
448 if (NVT == Res.getValueType()) {
449 // The high bits are not guaranteed to be anything. Insert an extend.
450 if (N->getOpcode() == ISD::SIGN_EXTEND)
451 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
452 DAG.getValueType(N->getOperand(0).getValueType()));
453 if (N->getOpcode() == ISD::ZERO_EXTEND)
454 return DAG.getZeroExtendInReg(Res, dl,
455 N->getOperand(0).getValueType().getScalarType());
456 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
461 // Otherwise, just extend the original operand all the way to the larger type.
462 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
465 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
466 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
467 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
468 ISD::LoadExtType ExtType =
469 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
471 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
472 N->getMemoryVT(), N->getMemOperand());
474 // Legalize the chain result - switch anything that used the old chain to
476 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
480 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
481 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
482 SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
485 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
486 N->getMask(), ExtSrc0, N->getMemoryVT(),
487 N->getMemOperand(), ISD::SEXTLOAD);
488 // Legalize the chain result - switch anything that used the old chain to
490 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
494 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
495 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
496 SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
497 assert(NVT == ExtSrc0.getValueType() &&
498 "Gather result type and the passThru agrument type should be the same");
501 SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
503 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
504 N->getMemoryVT(), dl, Ops,
506 // Legalize the chain result - switch anything that used the old chain to
508 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
512 /// Promote the overflow flag of an overflowing arithmetic node.
513 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
514 // Simply change the return type of the boolean result.
515 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
516 EVT ValueVTs[] = { N->getValueType(0), NVT };
517 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
518 unsigned NumOps = N->getNumOperands();
519 assert(NumOps <= 3 && "Too many operands");
521 Ops[2] = N->getOperand(2);
523 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
524 DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
526 // Modified the sum result - switch anything that used the old sum to use
528 ReplaceValueWith(SDValue(N, 0), Res);
530 return SDValue(Res.getNode(), 1);
533 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
535 return PromoteIntRes_Overflow(N);
537 // The operation overflowed iff the result in the larger type is not the
538 // sign extension of its truncation to the original type.
539 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
540 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
541 EVT OVT = N->getOperand(0).getValueType();
542 EVT NVT = LHS.getValueType();
545 // Do the arithmetic in the larger type.
546 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
547 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
549 // Calculate the overflow flag: sign extend the arithmetic result from
550 // the original type.
551 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
552 DAG.getValueType(OVT));
553 // Overflowed if and only if this is not equal to Res.
554 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
556 // Use the calculated overflow everywhere.
557 ReplaceValueWith(SDValue(N, 1), Ofl);
562 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
563 SDValue LHS = GetPromotedInteger(N->getOperand(1));
564 SDValue RHS = GetPromotedInteger(N->getOperand(2));
565 return DAG.getSelect(SDLoc(N),
566 LHS.getValueType(), N->getOperand(0), LHS, RHS);
569 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
570 SDValue Mask = N->getOperand(0);
571 EVT OpTy = N->getOperand(1).getValueType();
573 // Promote all the way up to the canonical SetCC type.
574 Mask = PromoteTargetBoolean(Mask, OpTy);
575 SDValue LHS = GetPromotedInteger(N->getOperand(1));
576 SDValue RHS = GetPromotedInteger(N->getOperand(2));
577 return DAG.getNode(ISD::VSELECT, SDLoc(N),
578 LHS.getValueType(), Mask, LHS, RHS);
581 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
582 SDValue LHS = GetPromotedInteger(N->getOperand(2));
583 SDValue RHS = GetPromotedInteger(N->getOperand(3));
584 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
585 LHS.getValueType(), N->getOperand(0),
586 N->getOperand(1), LHS, RHS, N->getOperand(4));
589 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
590 EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
592 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
594 // Only use the result of getSetCCResultType if it is legal,
595 // otherwise just use the promoted result type (NVT).
596 if (!TLI.isTypeLegal(SVT))
600 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
601 "Vector compare must return a vector result!");
603 SDValue LHS = N->getOperand(0);
604 SDValue RHS = N->getOperand(1);
605 if (LHS.getValueType() != RHS.getValueType()) {
606 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
607 !LHS.getValueType().isVector())
608 LHS = GetPromotedInteger(LHS);
609 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
610 !RHS.getValueType().isVector())
611 RHS = GetPromotedInteger(RHS);
614 // Get the SETCC result using the canonical SETCC type.
615 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
618 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
619 // Convert to the expected type.
620 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
623 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
624 SDValue LHS = N->getOperand(0);
625 SDValue RHS = N->getOperand(1);
626 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
627 LHS = GetPromotedInteger(LHS);
628 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
629 RHS = ZExtPromotedInteger(RHS);
630 return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
633 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
634 SDValue Op = GetPromotedInteger(N->getOperand(0));
635 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
636 Op.getValueType(), Op, N->getOperand(1));
639 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
640 // The input may have strange things in the top bits of the registers, but
641 // these operations don't care. They may have weird bits going out, but
642 // that too is okay if they are integer operations.
643 SDValue LHS = GetPromotedInteger(N->getOperand(0));
644 SDValue RHS = GetPromotedInteger(N->getOperand(1));
645 return DAG.getNode(N->getOpcode(), SDLoc(N),
646 LHS.getValueType(), LHS, RHS);
649 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
650 // Sign extend the input.
651 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
652 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
653 return DAG.getNode(N->getOpcode(), SDLoc(N),
654 LHS.getValueType(), LHS, RHS);
657 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
658 // Zero extend the input.
659 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
660 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
661 return DAG.getNode(N->getOpcode(), SDLoc(N),
662 LHS.getValueType(), LHS, RHS);
665 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
666 SDValue LHS = N->getOperand(0);
667 SDValue RHS = N->getOperand(1);
668 // The input value must be properly sign extended.
669 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
670 LHS = SExtPromotedInteger(LHS);
671 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
672 RHS = ZExtPromotedInteger(RHS);
673 return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
676 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
677 SDValue LHS = N->getOperand(0);
678 SDValue RHS = N->getOperand(1);
679 // The input value must be properly zero extended.
680 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
681 LHS = ZExtPromotedInteger(LHS);
682 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
683 RHS = ZExtPromotedInteger(RHS);
684 return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
687 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
688 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
690 SDValue InOp = N->getOperand(0);
693 switch (getTypeAction(InOp.getValueType())) {
694 default: llvm_unreachable("Unknown type action!");
695 case TargetLowering::TypeLegal:
696 case TargetLowering::TypeExpandInteger:
699 case TargetLowering::TypePromoteInteger:
700 Res = GetPromotedInteger(InOp);
702 case TargetLowering::TypeSplitVector: {
703 EVT InVT = InOp.getValueType();
704 assert(InVT.isVector() && "Cannot split scalar types");
705 unsigned NumElts = InVT.getVectorNumElements();
706 assert(NumElts == NVT.getVectorNumElements() &&
707 "Dst and Src must have the same number of elements");
708 assert(isPowerOf2_32(NumElts) &&
709 "Promoted vector type must be a power of two");
712 GetSplitVector(InOp, EOp1, EOp2);
714 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
716 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
717 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
719 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
721 case TargetLowering::TypeWidenVector: {
722 SDValue WideInOp = GetWidenedVector(InOp);
724 // Truncate widened InOp.
725 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
726 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
727 N->getValueType(0).getScalarType(), NumElem);
728 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
730 // Zero extend so that the elements are of same type as those of NVT
731 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
733 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
735 // Extract the low NVT subvector.
736 MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
737 SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
738 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
742 // Truncate to NVT instead of VT
743 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
746 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
748 return PromoteIntRes_Overflow(N);
750 // The operation overflowed iff the result in the larger type is not the
751 // zero extension of its truncation to the original type.
752 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
753 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
754 EVT OVT = N->getOperand(0).getValueType();
755 EVT NVT = LHS.getValueType();
758 // Do the arithmetic in the larger type.
759 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
760 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
762 // Calculate the overflow flag: zero extend the arithmetic result from
763 // the original type.
764 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
765 // Overflowed if and only if this is not equal to Res.
766 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
768 // Use the calculated overflow everywhere.
769 ReplaceValueWith(SDValue(N, 1), Ofl);
774 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
776 return PromoteIntRes_Overflow(N);
777 llvm_unreachable("Not implemented");
780 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
781 // Promote the overflow bit trivially.
783 return PromoteIntRes_Overflow(N);
785 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
787 EVT SmallVT = LHS.getValueType();
789 // To determine if the result overflowed in a larger type, we extend the
790 // input to the larger type, do the multiply (checking if it overflows),
791 // then also check the high bits of the result to see if overflow happened
793 if (N->getOpcode() == ISD::SMULO) {
794 LHS = SExtPromotedInteger(LHS);
795 RHS = SExtPromotedInteger(RHS);
797 LHS = ZExtPromotedInteger(LHS);
798 RHS = ZExtPromotedInteger(RHS);
800 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
801 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
803 // Overflow occurred if it occurred in the larger type, or if the high part
804 // of the result does not zero/sign-extend the low part. Check this second
805 // possibility first.
807 if (N->getOpcode() == ISD::UMULO) {
808 // Unsigned overflow occurred if the high part is non-zero.
809 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
810 DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
812 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
813 DAG.getConstant(0, DL, Hi.getValueType()),
816 // Signed overflow occurred if the high part does not sign extend the low.
817 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
818 Mul, DAG.getValueType(SmallVT));
819 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
822 // The only other way for overflow to occur is if the multiplication in the
823 // larger type itself overflowed.
824 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
825 SDValue(Mul.getNode(), 1));
827 // Use the calculated overflow everywhere.
828 ReplaceValueWith(SDValue(N, 1), Overflow);
832 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
833 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
834 N->getValueType(0)));
837 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
838 SDValue Chain = N->getOperand(0); // Get the chain.
839 SDValue Ptr = N->getOperand(1); // Get the pointer.
840 EVT VT = N->getValueType(0);
843 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
844 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
845 // The argument is passed as NumRegs registers of type RegVT.
847 SmallVector<SDValue, 8> Parts(NumRegs);
848 for (unsigned i = 0; i < NumRegs; ++i) {
849 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
850 N->getConstantOperandVal(3));
851 Chain = Parts[i].getValue(1);
854 // Handle endianness of the load.
855 if (DAG.getDataLayout().isBigEndian())
856 std::reverse(Parts.begin(), Parts.end());
858 // Assemble the parts in the promoted type.
859 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
860 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
861 for (unsigned i = 1; i < NumRegs; ++i) {
862 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
863 // Shift it to the right position and "or" it in.
864 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
865 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
866 TLI.getPointerTy(DAG.getDataLayout())));
867 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
870 // Modified the chain result - switch anything that used the old chain to
872 ReplaceValueWith(SDValue(N, 1), Chain);
877 //===----------------------------------------------------------------------===//
878 // Integer Operand Promotion
879 //===----------------------------------------------------------------------===//
881 /// PromoteIntegerOperand - This method is called when the specified operand of
882 /// the specified node is found to need promotion. At this point, all of the
883 /// result types of the node are known to be legal, but other operands of the
884 /// node may need promotion or expansion as well as the specified one.
885 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
886 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
887 SDValue Res = SDValue();
889 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
892 switch (N->getOpcode()) {
895 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
896 N->dump(&DAG); dbgs() << "\n";
898 llvm_unreachable("Do not know how to promote this operator's operand!");
900 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
901 case ISD::ATOMIC_STORE:
902 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
904 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
905 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
906 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
907 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
908 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
909 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
910 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
911 case ISD::INSERT_VECTOR_ELT:
912 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
913 case ISD::SCALAR_TO_VECTOR:
914 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
916 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
917 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
918 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
919 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
920 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
921 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
923 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
925 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
927 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
929 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
931 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
932 case ISD::FP16_TO_FP:
933 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
934 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
935 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
941 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
944 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
947 // If the result is null, the sub-method took care of registering results etc.
948 if (!Res.getNode()) return false;
950 // If the result is N, the sub-method updated N in place. Tell the legalizer
952 if (Res.getNode() == N)
955 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
956 "Invalid operand expansion");
958 ReplaceValueWith(SDValue(N, 0), Res);
962 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
963 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
964 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
965 ISD::CondCode CCCode) {
966 // We have to insert explicit sign or zero extends. Note that we could
967 // insert sign extends for ALL conditions, but zero extend is cheaper on
968 // many machines (an AND instead of two shifts), so prefer it.
970 default: llvm_unreachable("Unknown integer comparison!");
973 SDValue OpL = GetPromotedInteger(NewLHS);
974 SDValue OpR = GetPromotedInteger(NewRHS);
976 // We would prefer to promote the comparison operand with sign extension.
977 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
978 // than the width of NewLHS/NewRH, we can avoid inserting real truncate
979 // instruction, which is redudant eventually.
980 unsigned OpLEffectiveBits =
981 OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
982 unsigned OpREffectiveBits =
983 OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
984 if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() &&
985 OpREffectiveBits <= NewRHS.getValueSizeInBits()) {
989 NewLHS = ZExtPromotedInteger(NewLHS);
990 NewRHS = ZExtPromotedInteger(NewRHS);
998 // ALL of these operations will work if we either sign or zero extend
999 // the operands (including the unsigned comparisons!). Zero extend is
1000 // usually a simpler/cheaper operation, so prefer it.
1001 NewLHS = ZExtPromotedInteger(NewLHS);
1002 NewRHS = ZExtPromotedInteger(NewRHS);
1008 NewLHS = SExtPromotedInteger(NewLHS);
1009 NewRHS = SExtPromotedInteger(NewRHS);
1014 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1015 SDValue Op = GetPromotedInteger(N->getOperand(0));
1016 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1019 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1020 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1021 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1022 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1025 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1026 // This should only occur in unusual situations like bitcasting to an
1027 // x86_fp80, so just turn it into a store+load
1028 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1031 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1032 assert(OpNo == 2 && "Don't know how to promote this operand!");
1034 SDValue LHS = N->getOperand(2);
1035 SDValue RHS = N->getOperand(3);
1036 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1038 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1040 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1041 N->getOperand(1), LHS, RHS, N->getOperand(4)),
1045 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1046 assert(OpNo == 1 && "only know how to promote condition");
1048 // Promote all the way up to the canonical SetCC type.
1049 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1051 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1052 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1053 N->getOperand(2)), 0);
1056 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1057 // Since the result type is legal, the operands must promote to it.
1058 EVT OVT = N->getOperand(0).getValueType();
1059 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1060 SDValue Hi = GetPromotedInteger(N->getOperand(1));
1061 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1064 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1065 DAG.getConstant(OVT.getSizeInBits(), dl,
1066 TLI.getPointerTy(DAG.getDataLayout())));
1067 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1070 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1071 // The vector type is legal but the element type is not. This implies
1072 // that the vector is a power-of-two in length and that the element
1073 // type does not have a strange size (eg: it is not i1).
1074 EVT VecVT = N->getValueType(0);
1075 unsigned NumElts = VecVT.getVectorNumElements();
1076 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1077 "Legal vector of one illegal element?");
1079 // Promote the inserted value. The type does not need to match the
1080 // vector element type. Check that any extra bits introduced will be
1082 assert(N->getOperand(0).getValueSizeInBits() >=
1083 N->getValueType(0).getScalarSizeInBits() &&
1084 "Type of inserted value narrower than vector element type!");
1086 SmallVector<SDValue, 16> NewOps;
1087 for (unsigned i = 0; i < NumElts; ++i)
1088 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1090 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1093 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1096 // Promote the inserted value. This is valid because the type does not
1097 // have to match the vector element type.
1099 // Check that any extra bits introduced will be truncated away.
1100 assert(N->getOperand(1).getValueSizeInBits() >=
1101 N->getValueType(0).getScalarSizeInBits() &&
1102 "Type of inserted value narrower than vector element type!");
1103 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1104 GetPromotedInteger(N->getOperand(1)),
1109 assert(OpNo == 2 && "Different operand and result vector types?");
1111 // Promote the index.
1112 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1113 TLI.getVectorIdxTy(DAG.getDataLayout()));
1114 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1115 N->getOperand(1), Idx), 0);
1118 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1119 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1120 // the operand in place.
1121 return SDValue(DAG.UpdateNodeOperands(N,
1122 GetPromotedInteger(N->getOperand(0))), 0);
1125 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1126 assert(OpNo == 0 && "Only know how to promote the condition!");
1127 SDValue Cond = N->getOperand(0);
1128 EVT OpTy = N->getOperand(1).getValueType();
1130 if (N->getOpcode() == ISD::VSELECT)
1131 if (SDValue Res = WidenVSELECTAndMask(N))
1134 // Promote all the way up to the canonical SetCC type.
1135 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1136 Cond = PromoteTargetBoolean(Cond, OpVT);
1138 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1139 N->getOperand(2)), 0);
1142 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1143 assert(OpNo == 0 && "Don't know how to promote this operand!");
1145 SDValue LHS = N->getOperand(0);
1146 SDValue RHS = N->getOperand(1);
1147 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1149 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1150 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1151 N->getOperand(3), N->getOperand(4)), 0);
1154 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1155 assert(OpNo == 0 && "Don't know how to promote this operand!");
1157 SDValue LHS = N->getOperand(0);
1158 SDValue RHS = N->getOperand(1);
1159 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1161 // The CC (#2) is always legal.
1162 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1165 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1166 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1167 ZExtPromotedInteger(N->getOperand(1))), 0);
1170 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1171 SDValue Op = GetPromotedInteger(N->getOperand(0));
1173 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1174 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1175 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1178 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1179 return SDValue(DAG.UpdateNodeOperands(N,
1180 SExtPromotedInteger(N->getOperand(0))), 0);
1183 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1184 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1185 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1188 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1190 // Truncate the value and store the result.
1191 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1192 N->getMemoryVT(), N->getMemOperand());
1195 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1198 SDValue DataOp = N->getValue();
1199 EVT DataVT = DataOp.getValueType();
1200 SDValue Mask = N->getMask();
1203 bool TruncateStore = false;
1205 // Mask comes before the data operand. If the data operand is legal, we just
1206 // promote the mask.
1207 // When the data operand has illegal type, we should legalize the data
1208 // operand first. The mask will be promoted/splitted/widened according to
1209 // the data operand type.
1210 if (TLI.isTypeLegal(DataVT))
1211 Mask = PromoteTargetBoolean(Mask, DataVT);
1213 if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
1214 return PromoteIntOp_MSTORE(N, 3);
1216 else if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
1217 return WidenVecOp_MSTORE(N, 3);
1220 assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
1221 return SplitVecOp_MSTORE(N, 3);
1224 } else { // Data operand
1225 assert(OpNo == 3 && "Unexpected operand for promotion");
1226 DataOp = GetPromotedInteger(DataOp);
1227 Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
1228 TruncateStore = true;
1231 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1232 N->getMemoryVT(), N->getMemOperand(),
1233 TruncateStore, N->isCompressingStore());
1236 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1238 assert(OpNo == 2 && "Only know how to promote the mask!");
1239 EVT DataVT = N->getValueType(0);
1240 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1241 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1242 NewOps[OpNo] = Mask;
1243 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1246 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1249 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1252 EVT DataVT = N->getValueType(0);
1253 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1255 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1257 SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1258 // updated in place.
1259 if (Res.getNode() == N)
1262 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
1263 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1267 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1269 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1272 EVT DataVT = N->getValue().getValueType();
1273 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1275 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1276 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1279 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1280 SDValue Op = GetPromotedInteger(N->getOperand(0));
1281 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1284 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1285 return SDValue(DAG.UpdateNodeOperands(N,
1286 ZExtPromotedInteger(N->getOperand(0))), 0);
1289 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1291 SDValue Op = GetPromotedInteger(N->getOperand(0));
1292 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1293 return DAG.getZeroExtendInReg(Op, dl,
1294 N->getOperand(0).getValueType().getScalarType());
1297 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1298 assert(OpNo == 2 && "Don't know how to promote this operand!");
1300 SDValue LHS = N->getOperand(0);
1301 SDValue RHS = N->getOperand(1);
1302 SDValue Carry = N->getOperand(2);
1305 auto VT = getSetCCResultType(LHS.getValueType());
1306 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT);
1308 case TargetLoweringBase::UndefinedBooleanContent:
1309 Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1311 case TargetLoweringBase::ZeroOrOneBooleanContent:
1312 Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1314 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
1315 Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1319 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1322 //===----------------------------------------------------------------------===//
1323 // Integer Result Expansion
1324 //===----------------------------------------------------------------------===//
1326 /// ExpandIntegerResult - This method is called when the specified result of the
1327 /// specified node is found to need expansion. At this point, the node may also
1328 /// have invalid operands or may have other results that need promotion, we just
1329 /// know that (at least) one result needs expansion.
1330 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1331 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1333 Lo = Hi = SDValue();
1335 // See if the target wants to custom expand this node.
1336 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1339 switch (N->getOpcode()) {
1342 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1343 N->dump(&DAG); dbgs() << "\n";
1345 llvm_unreachable("Do not know how to expand the result of this operator!");
1347 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1348 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1349 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1350 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1352 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1353 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1354 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1355 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1356 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1358 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1359 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1360 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1361 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1362 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1363 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1364 case ISD::CTLZ_ZERO_UNDEF:
1365 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1366 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1367 case ISD::CTTZ_ZERO_UNDEF:
1368 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1369 case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1370 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1371 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1372 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1373 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1374 case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1375 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1376 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1377 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1378 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1379 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1380 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1381 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1382 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1383 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1385 case ISD::ATOMIC_LOAD_ADD:
1386 case ISD::ATOMIC_LOAD_SUB:
1387 case ISD::ATOMIC_LOAD_AND:
1388 case ISD::ATOMIC_LOAD_OR:
1389 case ISD::ATOMIC_LOAD_XOR:
1390 case ISD::ATOMIC_LOAD_NAND:
1391 case ISD::ATOMIC_LOAD_MIN:
1392 case ISD::ATOMIC_LOAD_MAX:
1393 case ISD::ATOMIC_LOAD_UMIN:
1394 case ISD::ATOMIC_LOAD_UMAX:
1395 case ISD::ATOMIC_SWAP:
1396 case ISD::ATOMIC_CMP_SWAP: {
1397 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1398 SplitInteger(Tmp.first, Lo, Hi);
1399 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1402 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1403 AtomicSDNode *AN = cast<AtomicSDNode>(N);
1404 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1405 SDValue Tmp = DAG.getAtomicCmpSwap(
1406 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1407 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1408 AN->getMemOperand());
1410 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1411 // success simply by comparing the loaded value against the ingoing
1413 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1414 N->getOperand(2), ISD::SETEQ);
1416 SplitInteger(Tmp, Lo, Hi);
1417 ReplaceValueWith(SDValue(N, 1), Success);
1418 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1424 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1429 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1432 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1435 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1438 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1441 case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1445 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1448 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1450 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1452 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1455 // If Lo/Hi is null, the sub-method took care of registering results etc.
1457 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1460 /// Lower an atomic node to the appropriate builtin call.
1461 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1462 unsigned Opc = Node->getOpcode();
1463 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1464 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1465 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1467 return ExpandChainLibCall(LC, Node, false);
1470 /// N is a shift by a value that needs to be expanded,
1471 /// and the shift amount is a constant 'Amt'. Expand the operation.
1472 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1473 SDValue &Lo, SDValue &Hi) {
1475 // Expand the incoming operand to be shifted, so that we have its parts
1477 GetExpandedInteger(N->getOperand(0), InL, InH);
1479 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1480 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1487 EVT NVT = InL.getValueType();
1488 unsigned VTBits = N->getValueType(0).getSizeInBits();
1489 unsigned NVTBits = NVT.getSizeInBits();
1490 EVT ShTy = N->getOperand(1).getValueType();
1492 if (N->getOpcode() == ISD::SHL) {
1493 if (Amt.ugt(VTBits)) {
1494 Lo = Hi = DAG.getConstant(0, DL, NVT);
1495 } else if (Amt.ugt(NVTBits)) {
1496 Lo = DAG.getConstant(0, DL, NVT);
1497 Hi = DAG.getNode(ISD::SHL, DL,
1498 NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1499 } else if (Amt == NVTBits) {
1500 Lo = DAG.getConstant(0, DL, NVT);
1503 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1504 Hi = DAG.getNode(ISD::OR, DL, NVT,
1505 DAG.getNode(ISD::SHL, DL, NVT, InH,
1506 DAG.getConstant(Amt, DL, ShTy)),
1507 DAG.getNode(ISD::SRL, DL, NVT, InL,
1508 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1513 if (N->getOpcode() == ISD::SRL) {
1514 if (Amt.ugt(VTBits)) {
1515 Lo = Hi = DAG.getConstant(0, DL, NVT);
1516 } else if (Amt.ugt(NVTBits)) {
1517 Lo = DAG.getNode(ISD::SRL, DL,
1518 NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1519 Hi = DAG.getConstant(0, DL, NVT);
1520 } else if (Amt == NVTBits) {
1522 Hi = DAG.getConstant(0, DL, NVT);
1524 Lo = DAG.getNode(ISD::OR, DL, NVT,
1525 DAG.getNode(ISD::SRL, DL, NVT, InL,
1526 DAG.getConstant(Amt, DL, ShTy)),
1527 DAG.getNode(ISD::SHL, DL, NVT, InH,
1528 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1529 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1534 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1535 if (Amt.ugt(VTBits)) {
1536 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1537 DAG.getConstant(NVTBits - 1, DL, ShTy));
1538 } else if (Amt.ugt(NVTBits)) {
1539 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1540 DAG.getConstant(Amt - NVTBits, DL, ShTy));
1541 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1542 DAG.getConstant(NVTBits - 1, DL, ShTy));
1543 } else if (Amt == NVTBits) {
1545 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1546 DAG.getConstant(NVTBits - 1, DL, ShTy));
1548 Lo = DAG.getNode(ISD::OR, DL, NVT,
1549 DAG.getNode(ISD::SRL, DL, NVT, InL,
1550 DAG.getConstant(Amt, DL, ShTy)),
1551 DAG.getNode(ISD::SHL, DL, NVT, InH,
1552 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1553 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1557 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1558 /// this shift based on knowledge of the high bit of the shift amount. If we
1559 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1561 bool DAGTypeLegalizer::
1562 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1563 SDValue Amt = N->getOperand(1);
1564 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1565 EVT ShTy = Amt.getValueType();
1566 unsigned ShBits = ShTy.getScalarSizeInBits();
1567 unsigned NVTBits = NVT.getScalarSizeInBits();
1568 assert(isPowerOf2_32(NVTBits) &&
1569 "Expanded integer type size not a power of two!");
1572 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1574 DAG.computeKnownBits(N->getOperand(1), Known);
1576 // If we don't know anything about the high bits, exit.
1577 if (((Known.Zero|Known.One) & HighBitMask) == 0)
1580 // Get the incoming operand to be shifted.
1582 GetExpandedInteger(N->getOperand(0), InL, InH);
1584 // If we know that any of the high bits of the shift amount are one, then we
1585 // can do this as a couple of simple shifts.
1586 if (Known.One.intersects(HighBitMask)) {
1587 // Mask out the high bit, which we know is set.
1588 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1589 DAG.getConstant(~HighBitMask, dl, ShTy));
1591 switch (N->getOpcode()) {
1592 default: llvm_unreachable("Unknown shift");
1594 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1595 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1598 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1599 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1602 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1603 DAG.getConstant(NVTBits - 1, dl, ShTy));
1604 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1609 // If we know that all of the high bits of the shift amount are zero, then we
1610 // can do this as a couple of simple shifts.
1611 if (HighBitMask.isSubsetOf(Known.Zero)) {
1612 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1613 // shift if x is zero. We can use XOR here because x is known to be smaller
1615 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1616 DAG.getConstant(NVTBits - 1, dl, ShTy));
1619 switch (N->getOpcode()) {
1620 default: llvm_unreachable("Unknown shift");
1621 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1623 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1626 // When shifting right the arithmetic for Lo and Hi is swapped.
1627 if (N->getOpcode() != ISD::SHL)
1628 std::swap(InL, InH);
1630 // Use a little trick to get the bits that move from Lo to Hi. First
1631 // shift by one bit.
1632 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1633 // Then compute the remaining shift with amount-1.
1634 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1636 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1637 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1639 if (N->getOpcode() != ISD::SHL)
1647 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1649 bool DAGTypeLegalizer::
1650 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1651 SDValue Amt = N->getOperand(1);
1652 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1653 EVT ShTy = Amt.getValueType();
1654 unsigned NVTBits = NVT.getSizeInBits();
1655 assert(isPowerOf2_32(NVTBits) &&
1656 "Expanded integer type size not a power of two!");
1659 // Get the incoming operand to be shifted.
1661 GetExpandedInteger(N->getOperand(0), InL, InH);
1663 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1664 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1665 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1666 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1667 Amt, NVBitsNode, ISD::SETULT);
1668 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1669 Amt, DAG.getConstant(0, dl, ShTy),
1672 SDValue LoS, HiS, LoL, HiL;
1673 switch (N->getOpcode()) {
1674 default: llvm_unreachable("Unknown shift");
1676 // Short: ShAmt < NVTBits
1677 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1678 HiS = DAG.getNode(ISD::OR, dl, NVT,
1679 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1680 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1682 // Long: ShAmt >= NVTBits
1683 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1684 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1686 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1687 Hi = DAG.getSelect(dl, NVT, isZero, InH,
1688 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1691 // Short: ShAmt < NVTBits
1692 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1693 LoS = DAG.getNode(ISD::OR, dl, NVT,
1694 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1695 // FIXME: If Amt is zero, the following shift generates an undefined result
1696 // on some architectures.
1697 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1699 // Long: ShAmt >= NVTBits
1700 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1701 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1703 Lo = DAG.getSelect(dl, NVT, isZero, InL,
1704 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1705 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1708 // Short: ShAmt < NVTBits
1709 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1710 LoS = DAG.getNode(ISD::OR, dl, NVT,
1711 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1712 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1714 // Long: ShAmt >= NVTBits
1715 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1716 DAG.getConstant(NVTBits - 1, dl, ShTy));
1717 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1719 Lo = DAG.getSelect(dl, NVT, isZero, InL,
1720 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1721 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1726 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1729 default: llvm_unreachable("invalid min/max opcode");
1731 return std::make_pair(ISD::SETGT, ISD::UMAX);
1733 return std::make_pair(ISD::SETUGT, ISD::UMAX);
1735 return std::make_pair(ISD::SETLT, ISD::UMIN);
1737 return std::make_pair(ISD::SETULT, ISD::UMIN);
1741 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1742 SDValue &Lo, SDValue &Hi) {
1744 ISD::NodeType LoOpc;
1745 ISD::CondCode CondC;
1746 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1748 // Expand the subcomponents.
1749 SDValue LHSL, LHSH, RHSL, RHSH;
1750 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1751 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1754 EVT NVT = LHSL.getValueType();
1755 EVT CCT = getSetCCResultType(NVT);
1757 // Hi part is always the same op
1758 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1760 // We need to know whether to select Lo part that corresponds to 'winning'
1761 // Hi part or if Hi parts are equal.
1762 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1763 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1765 // Lo part corresponding to the 'winning' Hi part
1766 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1768 // Recursed Lo part if Hi parts are equal, this uses unsigned version
1769 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1771 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1774 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1775 SDValue &Lo, SDValue &Hi) {
1777 // Expand the subcomponents.
1778 SDValue LHSL, LHSH, RHSL, RHSH;
1779 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1780 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1782 EVT NVT = LHSL.getValueType();
1783 SDValue LoOps[2] = { LHSL, RHSL };
1784 SDValue HiOps[3] = { LHSH, RHSH };
1786 bool HasOpCarry = TLI.isOperationLegalOrCustom(
1787 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
1788 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1790 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1791 if (N->getOpcode() == ISD::ADD) {
1792 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1793 HiOps[2] = Lo.getValue(1);
1794 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1796 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1797 HiOps[2] = Lo.getValue(1);
1798 Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1803 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1804 // them. TODO: Teach operation legalization how to expand unsupported
1805 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1806 // a carry of type MVT::Glue, but there doesn't seem to be any way to
1807 // generate a value of this type in the expanded code sequence.
1809 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1810 ISD::ADDC : ISD::SUBC,
1811 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1814 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1815 if (N->getOpcode() == ISD::ADD) {
1816 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1817 HiOps[2] = Lo.getValue(1);
1818 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1820 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1821 HiOps[2] = Lo.getValue(1);
1822 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1828 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1829 ISD::UADDO : ISD::USUBO,
1830 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1832 EVT OvfVT = getSetCCResultType(NVT);
1833 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1834 TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
1836 if (N->getOpcode() == ISD::ADD) {
1838 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1839 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1842 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1843 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1845 SDValue OVF = Lo.getValue(1);
1848 case TargetLoweringBase::UndefinedBooleanContent:
1849 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1851 case TargetLoweringBase::ZeroOrOneBooleanContent:
1852 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1853 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1855 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
1856 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1857 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1862 if (N->getOpcode() == ISD::ADD) {
1863 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1864 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1865 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1867 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1868 DAG.getConstant(1, dl, NVT),
1869 DAG.getConstant(0, dl, NVT));
1870 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1872 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1873 DAG.getConstant(1, dl, NVT), Carry1);
1874 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1876 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1877 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1879 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1880 LoOps[0], LoOps[1], ISD::SETULT);
1881 SDValue Borrow = DAG.getSelect(dl, NVT, Cmp,
1882 DAG.getConstant(1, dl, NVT),
1883 DAG.getConstant(0, dl, NVT));
1884 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1888 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1889 SDValue &Lo, SDValue &Hi) {
1890 // Expand the subcomponents.
1891 SDValue LHSL, LHSH, RHSL, RHSH;
1893 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1894 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1895 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1896 SDValue LoOps[2] = { LHSL, RHSL };
1897 SDValue HiOps[3] = { LHSH, RHSH };
1899 if (N->getOpcode() == ISD::ADDC) {
1900 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1901 HiOps[2] = Lo.getValue(1);
1902 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1904 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1905 HiOps[2] = Lo.getValue(1);
1906 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1909 // Legalized the flag result - switch anything that used the old flag to
1911 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1914 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1915 SDValue &Lo, SDValue &Hi) {
1916 // Expand the subcomponents.
1917 SDValue LHSL, LHSH, RHSL, RHSH;
1919 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1920 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1921 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1922 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1923 SDValue HiOps[3] = { LHSH, RHSH };
1925 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1926 HiOps[2] = Lo.getValue(1);
1927 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1929 // Legalized the flag result - switch anything that used the old flag to
1931 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1934 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1935 SDValue &Lo, SDValue &Hi) {
1936 SDValue LHS = N->getOperand(0);
1937 SDValue RHS = N->getOperand(1);
1942 bool HasOpCarry = TLI.isOperationLegalOrCustom(
1943 N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
1944 TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
1947 // Expand the subcomponents.
1948 SDValue LHSL, LHSH, RHSL, RHSH;
1949 GetExpandedInteger(LHS, LHSL, LHSH);
1950 GetExpandedInteger(RHS, RHSL, RHSH);
1951 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1952 SDValue LoOps[2] = { LHSL, RHSL };
1953 SDValue HiOps[3] = { LHSH, RHSH };
1955 unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
1956 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1957 HiOps[2] = Lo.getValue(1);
1958 Hi = DAG.getNode(Opc, dl, VTList, HiOps);
1960 Ovf = Hi.getValue(1);
1962 // Expand the result by simply replacing it with the equivalent
1963 // non-overflow-checking operation.
1964 auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1965 SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
1966 SplitInteger(Sum, Lo, Hi);
1968 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
1969 // overflows iff a - b > a.
1970 auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
1971 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
1974 // Legalized the flag result - switch anything that used the old flag to
1976 ReplaceValueWith(SDValue(N, 1), Ovf);
1979 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
1980 SDValue &Lo, SDValue &Hi) {
1981 // Expand the subcomponents.
1982 SDValue LHSL, LHSH, RHSL, RHSH;
1984 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1985 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1986 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1987 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1988 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
1990 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1991 HiOps[2] = Lo.getValue(1);
1992 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1994 // Legalized the flag result - switch anything that used the old flag to
1996 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1999 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2000 SDValue &Lo, SDValue &Hi) {
2001 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2003 SDValue Op = N->getOperand(0);
2004 if (Op.getValueType().bitsLE(NVT)) {
2005 // The low part is any extension of the input (which degenerates to a copy).
2006 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2007 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2009 // For example, extension of an i48 to an i64. The operand type necessarily
2010 // promotes to the result type, so will end up being expanded too.
2011 assert(getTypeAction(Op.getValueType()) ==
2012 TargetLowering::TypePromoteInteger &&
2013 "Only know how to promote this result!");
2014 SDValue Res = GetPromotedInteger(Op);
2015 assert(Res.getValueType() == N->getValueType(0) &&
2016 "Operand over promoted?");
2017 // Split the promoted operand. This will simplify when it is expanded.
2018 SplitInteger(Res, Lo, Hi);
2022 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2023 SDValue &Lo, SDValue &Hi) {
2025 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2026 EVT NVT = Lo.getValueType();
2027 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2028 unsigned NVTBits = NVT.getSizeInBits();
2029 unsigned EVTBits = EVT.getSizeInBits();
2031 if (NVTBits < EVTBits) {
2032 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2033 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2034 EVTBits - NVTBits)));
2036 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2037 // The high part replicates the sign bit of Lo, make it explicit.
2038 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2039 DAG.getConstant(NVTBits - 1, dl,
2040 TLI.getPointerTy(DAG.getDataLayout())));
2044 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2045 SDValue &Lo, SDValue &Hi) {
2047 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2048 EVT NVT = Lo.getValueType();
2049 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2050 unsigned NVTBits = NVT.getSizeInBits();
2051 unsigned EVTBits = EVT.getSizeInBits();
2053 if (NVTBits < EVTBits) {
2054 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2055 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2056 EVTBits - NVTBits)));
2058 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2059 // The high part must be zero, make it explicit.
2060 Hi = DAG.getConstant(0, dl, NVT);
2064 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2065 SDValue &Lo, SDValue &Hi) {
2067 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2068 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2069 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2072 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2073 SDValue &Lo, SDValue &Hi) {
2075 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2076 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2077 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2080 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2081 SDValue &Lo, SDValue &Hi) {
2082 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2083 unsigned NBitWidth = NVT.getSizeInBits();
2084 auto Constant = cast<ConstantSDNode>(N);
2085 const APInt &Cst = Constant->getAPIntValue();
2086 bool IsTarget = Constant->isTargetOpcode();
2087 bool IsOpaque = Constant->isOpaque();
2089 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2090 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2094 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2095 SDValue &Lo, SDValue &Hi) {
2097 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2098 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2099 EVT NVT = Lo.getValueType();
2101 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2102 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2104 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2105 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2107 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2108 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2109 DAG.getConstant(NVT.getSizeInBits(), dl,
2111 Hi = DAG.getConstant(0, dl, NVT);
2114 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2115 SDValue &Lo, SDValue &Hi) {
2117 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2118 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2119 EVT NVT = Lo.getValueType();
2120 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2121 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2122 Hi = DAG.getConstant(0, dl, NVT);
2125 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2126 SDValue &Lo, SDValue &Hi) {
2128 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2129 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2130 EVT NVT = Lo.getValueType();
2132 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2133 DAG.getConstant(0, dl, NVT), ISD::SETNE);
2135 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2136 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2138 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2139 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2140 DAG.getConstant(NVT.getSizeInBits(), dl,
2142 Hi = DAG.getConstant(0, dl, NVT);
2145 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2148 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2149 unsigned NBitWidth = NVT.getSizeInBits();
2151 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2152 Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2153 // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2154 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2155 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2158 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2161 EVT VT = N->getValueType(0);
2163 SDValue Op = N->getOperand(0);
2164 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2165 Op = GetPromotedFloat(Op);
2167 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
2168 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2169 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2173 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2176 EVT VT = N->getValueType(0);
2178 SDValue Op = N->getOperand(0);
2179 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2180 Op = GetPromotedFloat(Op);
2182 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
2183 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2184 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2188 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2189 SDValue &Lo, SDValue &Hi) {
2190 if (ISD::isNormalLoad(N)) {
2191 ExpandRes_NormalLoad(N, Lo, Hi);
2195 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2197 EVT VT = N->getValueType(0);
2198 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2199 SDValue Ch = N->getChain();
2200 SDValue Ptr = N->getBasePtr();
2201 ISD::LoadExtType ExtType = N->getExtensionType();
2202 unsigned Alignment = N->getAlignment();
2203 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2204 AAMDNodes AAInfo = N->getAAInfo();
2207 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2209 if (N->getMemoryVT().bitsLE(NVT)) {
2210 EVT MemVT = N->getMemoryVT();
2212 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2213 Alignment, MMOFlags, AAInfo);
2215 // Remember the chain.
2216 Ch = Lo.getValue(1);
2218 if (ExtType == ISD::SEXTLOAD) {
2219 // The high part is obtained by SRA'ing all but one of the bits of the
2221 unsigned LoSize = Lo.getValueSizeInBits();
2222 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2223 DAG.getConstant(LoSize - 1, dl,
2224 TLI.getPointerTy(DAG.getDataLayout())));
2225 } else if (ExtType == ISD::ZEXTLOAD) {
2226 // The high part is just a zero.
2227 Hi = DAG.getConstant(0, dl, NVT);
2229 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2230 // The high part is undefined.
2231 Hi = DAG.getUNDEF(NVT);
2233 } else if (DAG.getDataLayout().isLittleEndian()) {
2234 // Little-endian - low bits are at low addresses.
2235 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2238 unsigned ExcessBits =
2239 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2240 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2242 // Increment the pointer to the other half.
2243 unsigned IncrementSize = NVT.getSizeInBits()/8;
2244 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2245 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2246 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2247 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2248 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2250 // Build a factor node to remember that this load is independent of the
2252 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2255 // Big-endian - high bits are at low addresses. Favor aligned loads at
2256 // the cost of some bit-fiddling.
2257 EVT MemVT = N->getMemoryVT();
2258 unsigned EBytes = MemVT.getStoreSize();
2259 unsigned IncrementSize = NVT.getSizeInBits()/8;
2260 unsigned ExcessBits = (EBytes - IncrementSize)*8;
2262 // Load both the high bits and maybe some of the low bits.
2263 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2264 EVT::getIntegerVT(*DAG.getContext(),
2265 MemVT.getSizeInBits() - ExcessBits),
2266 Alignment, MMOFlags, AAInfo);
2268 // Increment the pointer to the other half.
2269 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2270 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2271 // Load the rest of the low bits.
2272 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2273 N->getPointerInfo().getWithOffset(IncrementSize),
2274 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2275 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2277 // Build a factor node to remember that this load is independent of the
2279 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2282 if (ExcessBits < NVT.getSizeInBits()) {
2283 // Transfer low bits from the bottom of Hi to the top of Lo.
2285 ISD::OR, dl, NVT, Lo,
2286 DAG.getNode(ISD::SHL, dl, NVT, Hi,
2287 DAG.getConstant(ExcessBits, dl,
2288 TLI.getPointerTy(DAG.getDataLayout()))));
2289 // Move high bits to the right position in Hi.
2290 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2292 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2293 TLI.getPointerTy(DAG.getDataLayout())));
2297 // Legalize the chain result - switch anything that used the old chain to
2299 ReplaceValueWith(SDValue(N, 1), Ch);
2302 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2303 SDValue &Lo, SDValue &Hi) {
2305 SDValue LL, LH, RL, RH;
2306 GetExpandedInteger(N->getOperand(0), LL, LH);
2307 GetExpandedInteger(N->getOperand(1), RL, RH);
2308 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2309 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2312 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2313 SDValue &Lo, SDValue &Hi) {
2314 EVT VT = N->getValueType(0);
2315 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2318 SDValue LL, LH, RL, RH;
2319 GetExpandedInteger(N->getOperand(0), LL, LH);
2320 GetExpandedInteger(N->getOperand(1), RL, RH);
2322 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2323 TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
2327 // If nothing else, we can make a libcall.
2328 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2330 LC = RTLIB::MUL_I16;
2331 else if (VT == MVT::i32)
2332 LC = RTLIB::MUL_I32;
2333 else if (VT == MVT::i64)
2334 LC = RTLIB::MUL_I64;
2335 else if (VT == MVT::i128)
2336 LC = RTLIB::MUL_I128;
2338 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2339 // We'll expand the multiplication by brute force because we have no other
2340 // options. This is a trivially-generalized version of the code from
2341 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2343 unsigned Bits = NVT.getSizeInBits();
2344 unsigned HalfBits = Bits >> 1;
2345 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2347 SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2348 SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2350 SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2351 SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2353 EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2354 if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2355 // The type from TLI is too small to fit the shift amount we want.
2356 // Override it with i32. The shift will have to be legalized.
2357 ShiftAmtTy = MVT::i32;
2359 SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2360 SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2361 SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2362 SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2364 SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2365 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2366 SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2367 SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2369 SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2370 DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2371 SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2373 SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2374 DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2375 DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2376 Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2377 DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2379 Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2380 DAG.getNode(ISD::ADD, dl, NVT,
2381 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2382 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2386 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2387 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2391 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2394 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2395 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2396 SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2399 ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2402 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2403 SDValue &Lo, SDValue &Hi) {
2404 SDValue LHS = Node->getOperand(0);
2405 SDValue RHS = Node->getOperand(1);
2408 // Expand the result by simply replacing it with the equivalent
2409 // non-overflow-checking operation.
2410 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2411 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2413 SplitInteger(Sum, Lo, Hi);
2415 // Compute the overflow.
2417 // LHSSign -> LHS >= 0
2418 // RHSSign -> RHS >= 0
2419 // SumSign -> Sum >= 0
2422 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2424 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2426 EVT OType = Node->getValueType(1);
2427 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2429 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2430 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2431 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2432 Node->getOpcode() == ISD::SADDO ?
2433 ISD::SETEQ : ISD::SETNE);
2435 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2436 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2438 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2440 // Use the calculated overflow everywhere.
2441 ReplaceValueWith(SDValue(Node, 1), Cmp);
2444 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2445 SDValue &Lo, SDValue &Hi) {
2446 EVT VT = N->getValueType(0);
2448 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2450 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
2451 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2452 SplitInteger(Res.getValue(0), Lo, Hi);
2456 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2458 LC = RTLIB::SDIV_I16;
2459 else if (VT == MVT::i32)
2460 LC = RTLIB::SDIV_I32;
2461 else if (VT == MVT::i64)
2462 LC = RTLIB::SDIV_I64;
2463 else if (VT == MVT::i128)
2464 LC = RTLIB::SDIV_I128;
2465 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2467 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2470 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2471 SDValue &Lo, SDValue &Hi) {
2472 EVT VT = N->getValueType(0);
2475 // If we can emit an efficient shift operation, do so now. Check to see if
2476 // the RHS is a constant.
2477 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2478 return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2480 // If we can determine that the high bit of the shift is zero or one, even if
2481 // the low bits are variable, emit this shift in an optimized form.
2482 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2485 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2487 if (N->getOpcode() == ISD::SHL) {
2488 PartsOpc = ISD::SHL_PARTS;
2489 } else if (N->getOpcode() == ISD::SRL) {
2490 PartsOpc = ISD::SRL_PARTS;
2492 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2493 PartsOpc = ISD::SRA_PARTS;
2496 // Next check to see if the target supports this SHL_PARTS operation or if it
2497 // will custom expand it.
2498 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2499 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2500 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2501 Action == TargetLowering::Custom) {
2502 // Expand the subcomponents.
2504 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2505 EVT VT = LHSL.getValueType();
2507 // If the shift amount operand is coming from a vector legalization it may
2508 // have an illegal type. Fix that first by casting the operand, otherwise
2509 // the new SHL_PARTS operation would need further legalization.
2510 SDValue ShiftOp = N->getOperand(1);
2511 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2512 assert(ShiftTy.getScalarSizeInBits() >=
2513 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2514 "ShiftAmountTy is too small to cover the range of this type!");
2515 if (ShiftOp.getValueType() != ShiftTy)
2516 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2518 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2519 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2520 Hi = Lo.getValue(1);
2524 // Otherwise, emit a libcall.
2525 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2527 if (N->getOpcode() == ISD::SHL) {
2528 isSigned = false; /*sign irrelevant*/
2530 LC = RTLIB::SHL_I16;
2531 else if (VT == MVT::i32)
2532 LC = RTLIB::SHL_I32;
2533 else if (VT == MVT::i64)
2534 LC = RTLIB::SHL_I64;
2535 else if (VT == MVT::i128)
2536 LC = RTLIB::SHL_I128;
2537 } else if (N->getOpcode() == ISD::SRL) {
2540 LC = RTLIB::SRL_I16;
2541 else if (VT == MVT::i32)
2542 LC = RTLIB::SRL_I32;
2543 else if (VT == MVT::i64)
2544 LC = RTLIB::SRL_I64;
2545 else if (VT == MVT::i128)
2546 LC = RTLIB::SRL_I128;
2548 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2551 LC = RTLIB::SRA_I16;
2552 else if (VT == MVT::i32)
2553 LC = RTLIB::SRA_I32;
2554 else if (VT == MVT::i64)
2555 LC = RTLIB::SRA_I64;
2556 else if (VT == MVT::i128)
2557 LC = RTLIB::SRA_I128;
2560 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2561 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2562 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2566 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2567 llvm_unreachable("Unsupported shift!");
2570 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2571 SDValue &Lo, SDValue &Hi) {
2572 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2574 SDValue Op = N->getOperand(0);
2575 if (Op.getValueType().bitsLE(NVT)) {
2576 // The low part is sign extension of the input (degenerates to a copy).
2577 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2578 // The high part is obtained by SRA'ing all but one of the bits of low part.
2579 unsigned LoSize = NVT.getSizeInBits();
2581 ISD::SRA, dl, NVT, Lo,
2582 DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2584 // For example, extension of an i48 to an i64. The operand type necessarily
2585 // promotes to the result type, so will end up being expanded too.
2586 assert(getTypeAction(Op.getValueType()) ==
2587 TargetLowering::TypePromoteInteger &&
2588 "Only know how to promote this result!");
2589 SDValue Res = GetPromotedInteger(Op);
2590 assert(Res.getValueType() == N->getValueType(0) &&
2591 "Operand over promoted?");
2592 // Split the promoted operand. This will simplify when it is expanded.
2593 SplitInteger(Res, Lo, Hi);
2594 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2595 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2596 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2601 void DAGTypeLegalizer::
2602 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2604 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2605 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2607 if (EVT.bitsLE(Lo.getValueType())) {
2608 // sext_inreg the low part if needed.
2609 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2612 // The high part gets the sign extension from the lo-part. This handles
2613 // things like sextinreg V:i64 from i8.
2614 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2615 DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2616 TLI.getPointerTy(DAG.getDataLayout())));
2618 // For example, extension of an i48 to an i64. Leave the low part alone,
2619 // sext_inreg the high part.
2620 unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2621 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2622 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2627 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2628 SDValue &Lo, SDValue &Hi) {
2629 EVT VT = N->getValueType(0);
2631 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2633 if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
2634 SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2635 SplitInteger(Res.getValue(1), Lo, Hi);
2639 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2641 LC = RTLIB::SREM_I16;
2642 else if (VT == MVT::i32)
2643 LC = RTLIB::SREM_I32;
2644 else if (VT == MVT::i64)
2645 LC = RTLIB::SREM_I64;
2646 else if (VT == MVT::i128)
2647 LC = RTLIB::SREM_I128;
2648 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2650 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2653 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2654 SDValue &Lo, SDValue &Hi) {
2655 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2657 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2658 Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2660 DAG.getConstant(NVT.getSizeInBits(), dl,
2661 TLI.getPointerTy(DAG.getDataLayout())));
2662 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2665 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2666 SDValue &Lo, SDValue &Hi) {
2667 EVT VT = N->getValueType(0);
2670 // A divide for UMULO should be faster than a function call.
2671 if (N->getOpcode() == ISD::UMULO) {
2672 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2674 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2675 SplitInteger(MUL, Lo, Hi);
2677 // A divide for UMULO will be faster than a function call. Select to
2678 // make sure we aren't using 0.
2679 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2680 RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2681 SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2682 DAG.getConstant(1, dl, VT), RHS);
2683 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2684 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2686 Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2687 DAG.getConstant(0, dl, N->getValueType(1)),
2689 ReplaceValueWith(SDValue(N, 1), Overflow);
2693 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2694 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2695 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2697 // Replace this with a libcall that will check overflow.
2698 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2700 LC = RTLIB::MULO_I32;
2701 else if (VT == MVT::i64)
2702 LC = RTLIB::MULO_I64;
2703 else if (VT == MVT::i128)
2704 LC = RTLIB::MULO_I128;
2705 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2707 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2708 // Temporary for the overflow value, default it to zero.
2710 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2711 MachinePointerInfo());
2713 TargetLowering::ArgListTy Args;
2714 TargetLowering::ArgListEntry Entry;
2715 for (const SDValue &Op : N->op_values()) {
2716 EVT ArgVT = Op.getValueType();
2717 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2720 Entry.IsSExt = true;
2721 Entry.IsZExt = false;
2722 Args.push_back(Entry);
2725 // Also pass the address of the overflow check.
2727 Entry.Ty = PtrTy->getPointerTo();
2728 Entry.IsSExt = true;
2729 Entry.IsZExt = false;
2730 Args.push_back(Entry);
2732 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2734 TargetLowering::CallLoweringInfo CLI(DAG);
2737 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2740 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2742 SplitInteger(CallInfo.first, Lo, Hi);
2744 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2745 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2746 DAG.getConstant(0, dl, PtrVT),
2748 // Use the overflow from the libcall everywhere.
2749 ReplaceValueWith(SDValue(N, 1), Ofl);
2752 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2753 SDValue &Lo, SDValue &Hi) {
2754 EVT VT = N->getValueType(0);
2756 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2758 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
2759 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2760 SplitInteger(Res.getValue(0), Lo, Hi);
2764 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2766 LC = RTLIB::UDIV_I16;
2767 else if (VT == MVT::i32)
2768 LC = RTLIB::UDIV_I32;
2769 else if (VT == MVT::i64)
2770 LC = RTLIB::UDIV_I64;
2771 else if (VT == MVT::i128)
2772 LC = RTLIB::UDIV_I128;
2773 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2775 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2778 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2779 SDValue &Lo, SDValue &Hi) {
2780 EVT VT = N->getValueType(0);
2782 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2784 if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
2785 SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2786 SplitInteger(Res.getValue(1), Lo, Hi);
2790 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2792 LC = RTLIB::UREM_I16;
2793 else if (VT == MVT::i32)
2794 LC = RTLIB::UREM_I32;
2795 else if (VT == MVT::i64)
2796 LC = RTLIB::UREM_I64;
2797 else if (VT == MVT::i128)
2798 LC = RTLIB::UREM_I128;
2799 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2801 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2804 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2805 SDValue &Lo, SDValue &Hi) {
2806 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2808 SDValue Op = N->getOperand(0);
2809 if (Op.getValueType().bitsLE(NVT)) {
2810 // The low part is zero extension of the input (degenerates to a copy).
2811 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2812 Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2814 // For example, extension of an i48 to an i64. The operand type necessarily
2815 // promotes to the result type, so will end up being expanded too.
2816 assert(getTypeAction(Op.getValueType()) ==
2817 TargetLowering::TypePromoteInteger &&
2818 "Only know how to promote this result!");
2819 SDValue Res = GetPromotedInteger(Op);
2820 assert(Res.getValueType() == N->getValueType(0) &&
2821 "Operand over promoted?");
2822 // Split the promoted operand. This will simplify when it is expanded.
2823 SplitInteger(Res, Lo, Hi);
2824 unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2825 Hi = DAG.getZeroExtendInReg(Hi, dl,
2826 EVT::getIntegerVT(*DAG.getContext(),
2831 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2832 SDValue &Lo, SDValue &Hi) {
2834 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2835 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2836 SDValue Zero = DAG.getConstant(0, dl, VT);
2837 SDValue Swap = DAG.getAtomicCmpSwap(
2838 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2839 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2840 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2842 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2843 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2846 //===----------------------------------------------------------------------===//
2847 // Integer Operand Expansion
2848 //===----------------------------------------------------------------------===//
2850 /// ExpandIntegerOperand - This method is called when the specified operand of
2851 /// the specified node is found to need expansion. At this point, all of the
2852 /// result types of the node are known to be legal, but other operands of the
2853 /// node may need promotion or expansion as well as the specified one.
2854 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2855 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2856 SDValue Res = SDValue();
2858 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2861 switch (N->getOpcode()) {
2864 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2865 N->dump(&DAG); dbgs() << "\n";
2867 llvm_unreachable("Do not know how to expand this operator's operand!");
2869 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2870 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2871 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2872 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2873 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2874 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2875 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2876 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2877 case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break;
2878 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2879 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2880 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2881 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2887 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2888 case ISD::RETURNADDR:
2889 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2891 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2894 // If the result is null, the sub-method took care of registering results etc.
2895 if (!Res.getNode()) return false;
2897 // If the result is N, the sub-method updated N in place. Tell the legalizer
2899 if (Res.getNode() == N)
2902 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2903 "Invalid operand expansion");
2905 ReplaceValueWith(SDValue(N, 0), Res);
2909 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2910 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2911 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2913 ISD::CondCode &CCCode,
2915 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2916 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2917 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2919 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2920 if (RHSLo == RHSHi) {
2921 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2922 if (RHSCST->isAllOnesValue()) {
2923 // Equality comparison to -1.
2924 NewLHS = DAG.getNode(ISD::AND, dl,
2925 LHSLo.getValueType(), LHSLo, LHSHi);
2932 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2933 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2934 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2935 NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2939 // If this is a comparison of the sign bit, just look at the top part.
2941 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2942 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2943 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2949 // FIXME: This generated code sucks.
2950 ISD::CondCode LowCC;
2952 default: llvm_unreachable("Unknown integer setcc!");
2954 case ISD::SETULT: LowCC = ISD::SETULT; break;
2956 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2958 case ISD::SETULE: LowCC = ISD::SETULE; break;
2960 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2963 // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
2964 // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
2965 // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
2967 // NOTE: on targets without efficient SELECT of bools, we can always use
2968 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
2969 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
2971 SDValue LoCmp, HiCmp;
2972 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
2973 TLI.isTypeLegal(RHSLo.getValueType()))
2974 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
2975 RHSLo, LowCC, false, DagCombineInfo, dl);
2976 if (!LoCmp.getNode())
2977 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2979 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
2980 TLI.isTypeLegal(RHSHi.getValueType()))
2981 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
2982 RHSHi, CCCode, false, DagCombineInfo, dl);
2983 if (!HiCmp.getNode())
2985 DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
2986 LHSHi, RHSHi, DAG.getCondCode(CCCode));
2988 ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
2989 ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
2991 bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
2992 CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
2994 if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
2995 (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
2996 (LoCmpC && LoCmpC->isNullValue())))) {
2997 // For LE / GE, if high part is known false, ignore the low part.
2998 // For LT / GT: if low part is known false, return the high part.
2999 // if high part is known true, ignore the low part.
3005 if (LHSHi == RHSHi) {
3006 // Comparing the low bits is enough.
3012 // Lower with SETCCE if the target supports it.
3013 // FIXME: Make all targets support this, then remove the other lowering.
3014 if (TLI.getOperationAction(
3016 TLI.getTypeToExpandTo(*DAG.getContext(), LHSLo.getValueType())) ==
3017 TargetLowering::Custom) {
3018 // SETCCE can detect < and >= directly. For > and <=, flip operands and
3020 bool FlipOperands = false;
3022 case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3023 case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3024 case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3025 case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3029 std::swap(LHSLo, RHSLo);
3030 std::swap(LHSHi, RHSHi);
3032 // Perform a wide subtraction, feeding the carry from the low part into
3033 // SETCCE. The SETCCE operation is essentially looking at the high part of
3034 // the result of LHS - RHS. It is negative iff LHS < RHS. It is zero or
3035 // positive iff LHS >= RHS.
3036 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
3037 SDValue LowCmp = DAG.getNode(ISD::SUBC, dl, VTList, LHSLo, RHSLo);
3039 DAG.getNode(ISD::SETCCE, dl, getSetCCResultType(LHSLo.getValueType()),
3040 LHSHi, RHSHi, LowCmp.getValue(1), DAG.getCondCode(CCCode));
3046 NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
3047 LHSHi, RHSHi, ISD::SETEQ, false,
3048 DagCombineInfo, dl);
3049 if (!NewLHS.getNode())
3050 NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
3051 LHSHi, RHSHi, ISD::SETEQ);
3052 NewLHS = DAG.getSelect(dl, LoCmp.getValueType(),
3053 NewLHS, LoCmp, HiCmp);
3057 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3058 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3059 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3060 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3062 // If ExpandSetCCOperands returned a scalar, we need to compare the result
3063 // against zero to select between true and false values.
3064 if (!NewRHS.getNode()) {
3065 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3066 CCCode = ISD::SETNE;
3069 // Update N to have the operands specified.
3070 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3071 DAG.getCondCode(CCCode), NewLHS, NewRHS,
3072 N->getOperand(4)), 0);
3075 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3076 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3077 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3078 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3080 // If ExpandSetCCOperands returned a scalar, we need to compare the result
3081 // against zero to select between true and false values.
3082 if (!NewRHS.getNode()) {
3083 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3084 CCCode = ISD::SETNE;
3087 // Update N to have the operands specified.
3088 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3089 N->getOperand(2), N->getOperand(3),
3090 DAG.getCondCode(CCCode)), 0);
3093 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3094 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3095 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3096 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3098 // If ExpandSetCCOperands returned a scalar, use it.
3099 if (!NewRHS.getNode()) {
3100 assert(NewLHS.getValueType() == N->getValueType(0) &&
3101 "Unexpected setcc expansion!");
3105 // Otherwise, update N to have the operands specified.
3106 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3107 DAG.getCondCode(CCCode)), 0);
3110 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
3111 SDValue LHS = N->getOperand(0);
3112 SDValue RHS = N->getOperand(1);
3113 SDValue Carry = N->getOperand(2);
3114 SDValue Cond = N->getOperand(3);
3115 SDLoc dl = SDLoc(N);
3117 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3118 GetExpandedInteger(LHS, LHSLo, LHSHi);
3119 GetExpandedInteger(RHS, RHSLo, RHSHi);
3121 // Expand to a SUBE for the low part and a smaller SETCCE for the high.
3122 SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
3123 SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry);
3124 return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi,
3125 LowCmp.getValue(1), Cond);
3128 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3129 // The value being shifted is legal, but the shift amount is too big.
3130 // It follows that either the result of the shift is undefined, or the
3131 // upper half of the shift amount is zero. Just use the lower half.
3133 GetExpandedInteger(N->getOperand(1), Lo, Hi);
3134 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3137 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3138 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3139 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3140 // constant to valid type.
3142 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3143 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3146 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3147 SDValue Op = N->getOperand(0);
3148 EVT DstVT = N->getValueType(0);
3149 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
3150 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3151 "Don't know how to expand this SINT_TO_FP!");
3152 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3155 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3156 if (ISD::isNormalStore(N))
3157 return ExpandOp_NormalStore(N, OpNo);
3159 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3160 assert(OpNo == 1 && "Can only expand the stored value so far");
3162 EVT VT = N->getOperand(1).getValueType();
3163 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3164 SDValue Ch = N->getChain();
3165 SDValue Ptr = N->getBasePtr();
3166 unsigned Alignment = N->getAlignment();
3167 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3168 AAMDNodes AAInfo = N->getAAInfo();
3172 assert(NVT.isByteSized() && "Expanded type not byte sized!");
3174 if (N->getMemoryVT().bitsLE(NVT)) {
3175 GetExpandedInteger(N->getValue(), Lo, Hi);
3176 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3177 N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3180 if (DAG.getDataLayout().isLittleEndian()) {
3181 // Little-endian - low bits are at low addresses.
3182 GetExpandedInteger(N->getValue(), Lo, Hi);
3184 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3187 unsigned ExcessBits =
3188 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3189 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3191 // Increment the pointer to the other half.
3192 unsigned IncrementSize = NVT.getSizeInBits()/8;
3193 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3194 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3195 Hi = DAG.getTruncStore(
3196 Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3197 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3198 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3201 // Big-endian - high bits are at low addresses. Favor aligned stores at
3202 // the cost of some bit-fiddling.
3203 GetExpandedInteger(N->getValue(), Lo, Hi);
3205 EVT ExtVT = N->getMemoryVT();
3206 unsigned EBytes = ExtVT.getStoreSize();
3207 unsigned IncrementSize = NVT.getSizeInBits()/8;
3208 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3209 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3210 ExtVT.getSizeInBits() - ExcessBits);
3212 if (ExcessBits < NVT.getSizeInBits()) {
3213 // Transfer high bits from the top of Lo to the bottom of Hi.
3214 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3215 DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3216 TLI.getPointerTy(DAG.getDataLayout())));
3218 ISD::OR, dl, NVT, Hi,
3219 DAG.getNode(ISD::SRL, dl, NVT, Lo,
3220 DAG.getConstant(ExcessBits, dl,
3221 TLI.getPointerTy(DAG.getDataLayout()))));
3224 // Store both the high bits and maybe some of the low bits.
3225 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3228 // Increment the pointer to the other half.
3229 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3230 DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3231 // Store the lowest ExcessBits bits in the second half.
3232 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3233 N->getPointerInfo().getWithOffset(IncrementSize),
3234 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3235 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3236 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3239 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3241 GetExpandedInteger(N->getOperand(0), InL, InH);
3242 // Just truncate the low part of the source.
3243 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3246 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3247 SDValue Op = N->getOperand(0);
3248 EVT SrcVT = Op.getValueType();
3249 EVT DstVT = N->getValueType(0);
3252 // The following optimization is valid only if every value in SrcVT (when
3253 // treated as signed) is representable in DstVT. Check that the mantissa
3254 // size of DstVT is >= than the number of bits in SrcVT -1.
3255 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3256 if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3257 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){
3258 // Do a signed conversion then adjust the result.
3259 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3260 SignedConv = TLI.LowerOperation(SignedConv, DAG);
3262 // The result of the signed conversion needs adjusting if the 'sign bit' of
3263 // the incoming integer was set. To handle this, we dynamically test to see
3264 // if it is set, and, if so, add a fudge factor.
3266 const uint64_t F32TwoE32 = 0x4F800000ULL;
3267 const uint64_t F32TwoE64 = 0x5F800000ULL;
3268 const uint64_t F32TwoE128 = 0x7F800000ULL;
3271 if (SrcVT == MVT::i32)
3272 FF = APInt(32, F32TwoE32);
3273 else if (SrcVT == MVT::i64)
3274 FF = APInt(32, F32TwoE64);
3275 else if (SrcVT == MVT::i128)
3276 FF = APInt(32, F32TwoE128);
3278 llvm_unreachable("Unsupported UINT_TO_FP!");
3280 // Check whether the sign bit is set.
3282 GetExpandedInteger(Op, Lo, Hi);
3283 SDValue SignSet = DAG.getSetCC(dl,
3284 getSetCCResultType(Hi.getValueType()),
3286 DAG.getConstant(0, dl, Hi.getValueType()),
3289 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3291 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3292 TLI.getPointerTy(DAG.getDataLayout()));
3294 // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3295 SDValue Zero = DAG.getIntPtrConstant(0, dl);
3296 SDValue Four = DAG.getIntPtrConstant(4, dl);
3297 if (DAG.getDataLayout().isBigEndian())
3298 std::swap(Zero, Four);
3299 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3301 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3302 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3304 Alignment = std::min(Alignment, 4u);
3306 // Load the value out, extending it from f32 to the destination float type.
3307 // FIXME: Avoid the extend by constructing the right constant pool?
3308 SDValue Fudge = DAG.getExtLoad(
3309 ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3310 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
3312 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3315 // Otherwise, use a libcall.
3316 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3317 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3318 "Don't know how to expand this UINT_TO_FP!");
3319 return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3322 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3324 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3325 cast<AtomicSDNode>(N)->getMemoryVT(),
3327 N->getOperand(1), N->getOperand(2),
3328 cast<AtomicSDNode>(N)->getMemOperand());
3329 return Swap.getValue(1);
3333 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3334 SDValue InOp0 = N->getOperand(0);
3335 EVT InVT = InOp0.getValueType();
3337 EVT OutVT = N->getValueType(0);
3338 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3339 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3340 unsigned OutNumElems = OutVT.getVectorNumElements();
3341 EVT NOutVTElem = NOutVT.getVectorElementType();
3344 SDValue BaseIdx = N->getOperand(1);
3346 SmallVector<SDValue, 8> Ops;
3347 Ops.reserve(OutNumElems);
3348 for (unsigned i = 0; i != OutNumElems; ++i) {
3350 // Extract the element from the original vector.
3351 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3352 BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3353 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3354 InVT.getVectorElementType(), N->getOperand(0), Index);
3356 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3357 // Insert the converted element to the new vector.
3361 return DAG.getBuildVector(NOutVT, dl, Ops);
3365 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3366 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3367 EVT VT = N->getValueType(0);
3370 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3372 SDValue V0 = GetPromotedInteger(N->getOperand(0));
3373 SDValue V1 = GetPromotedInteger(N->getOperand(1));
3374 EVT OutVT = V0.getValueType();
3376 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3380 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3381 EVT OutVT = N->getValueType(0);
3382 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3383 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3384 unsigned NumElems = N->getNumOperands();
3385 EVT NOutVTElem = NOutVT.getVectorElementType();
3389 SmallVector<SDValue, 8> Ops;
3390 Ops.reserve(NumElems);
3391 for (unsigned i = 0; i != NumElems; ++i) {
3393 // BUILD_VECTOR integer operand types are allowed to be larger than the
3394 // result's element type. This may still be true after the promotion. For
3395 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3396 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3397 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3398 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3400 Op = N->getOperand(i);
3404 return DAG.getBuildVector(NOutVT, dl, Ops);
3407 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3411 assert(!N->getOperand(0).getValueType().isVector() &&
3412 "Input must be a scalar");
3414 EVT OutVT = N->getValueType(0);
3415 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3416 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3417 EVT NOutVTElem = NOutVT.getVectorElementType();
3419 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3421 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3424 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3427 EVT OutVT = N->getValueType(0);
3428 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3429 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3431 EVT InElemTy = OutVT.getVectorElementType();
3432 EVT OutElemTy = NOutVT.getVectorElementType();
3434 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3435 unsigned NumOutElem = NOutVT.getVectorNumElements();
3436 unsigned NumOperands = N->getNumOperands();
3437 assert(NumElem * NumOperands == NumOutElem &&
3438 "Unexpected number of elements");
3440 // Take the elements from the first vector.
3441 SmallVector<SDValue, 8> Ops(NumOutElem);
3442 for (unsigned i = 0; i < NumOperands; ++i) {
3443 SDValue Op = N->getOperand(i);
3444 for (unsigned j = 0; j < NumElem; ++j) {
3445 SDValue Ext = DAG.getNode(
3446 ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
3447 DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3448 Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3452 return DAG.getBuildVector(NOutVT, dl, Ops);
3455 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3456 EVT VT = N->getValueType(0);
3457 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3458 assert(NVT.isVector() && "This type must be promoted to a vector type");
3462 // For operands whose TypeAction is to promote, extend the promoted node
3463 // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3464 // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3466 if (getTypeAction(N->getOperand(0).getValueType())
3467 == TargetLowering::TypePromoteInteger) {
3470 switch(N->getOpcode()) {
3471 case ISD::SIGN_EXTEND_VECTOR_INREG:
3472 Promoted = SExtPromotedInteger(N->getOperand(0));
3474 case ISD::ZERO_EXTEND_VECTOR_INREG:
3475 Promoted = ZExtPromotedInteger(N->getOperand(0));
3477 case ISD::ANY_EXTEND_VECTOR_INREG:
3478 Promoted = GetPromotedInteger(N->getOperand(0));
3481 llvm_unreachable("Node has unexpected Opcode");
3483 return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3486 // Directly extend to the appropriate transform-to type.
3487 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3490 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3491 EVT OutVT = N->getValueType(0);
3492 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3493 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3495 EVT NOutVTElem = NOutVT.getVectorElementType();
3498 SDValue V0 = GetPromotedInteger(N->getOperand(0));
3500 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3501 NOutVTElem, N->getOperand(1));
3502 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3503 V0, ConvElem, N->getOperand(2));
3506 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3508 SDValue V0 = GetPromotedInteger(N->getOperand(0));
3509 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3510 TLI.getVectorIdxTy(DAG.getDataLayout()));
3511 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3512 V0->getValueType(0).getScalarType(), V0, V1);
3514 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3515 // element types. If this is the case then we need to expand the outgoing
3516 // value and not truncate it.
3517 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3520 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3522 SDValue V0 = GetPromotedInteger(N->getOperand(0));
3523 MVT InVT = V0.getValueType().getSimpleVT();
3524 MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3525 N->getValueType(0).getVectorNumElements());
3526 SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3527 return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3530 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3532 unsigned NumElems = N->getNumOperands();
3534 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3536 SmallVector<SDValue, 8> NewOps;
3537 NewOps.reserve(NumElems);
3539 // For each incoming vector
3540 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3541 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3542 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3543 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3545 for (unsigned i=0; i<NumElem; ++i) {
3546 // Extract element from incoming vector
3547 SDValue Ex = DAG.getNode(
3548 ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3549 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3550 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3551 NewOps.push_back(Tr);
3555 return DAG.getBuildVector(N->getValueType(0), dl, NewOps);