1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 #include "HexagonISelLowering.h"
11 #include "HexagonRegisterInfo.h"
12 #include "HexagonSubtarget.h"
13 #include "llvm/Support/CommandLine.h"
17 static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
18 static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
19 static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
20 static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
24 HexagonTargetLowering::initializeHVXLowering() {
25 if (Subtarget.useHVX64BOps()) {
26 addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass);
27 addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
28 addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
29 addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
30 addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
31 addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
32 // These "short" boolean vector types should be legal because
33 // they will appear as results of vector compares. If they were
34 // not legal, type legalization would try to make them legal
35 // and that would require using operations that do not use or
36 // produce such types. That, in turn, would imply using custom
37 // nodes, which would be unoptimizable by the DAG combiner.
38 // The idea is to rely on target-independent operations as much
40 addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
41 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
42 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
43 addRegisterClass(MVT::v512i1, &Hexagon::HvxQRRegClass);
44 } else if (Subtarget.useHVX128BOps()) {
45 addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass);
46 addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass);
47 addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass);
48 addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass);
49 addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
50 addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass);
51 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
52 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
53 addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
54 addRegisterClass(MVT::v1024i1, &Hexagon::HvxQRRegClass);
57 // Set up operation actions.
59 bool Use64b = Subtarget.useHVX64BOps();
60 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
61 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
62 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
63 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
65 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
66 setOperationAction(Opc, FromTy, Promote);
67 AddPromotedToType(Opc, FromTy, ToTy);
70 setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal);
71 setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal);
73 for (MVT T : LegalV) {
74 setIndexedLoadAction(ISD::POST_INC, T, Legal);
75 setIndexedStoreAction(ISD::POST_INC, T, Legal);
77 setOperationAction(ISD::AND, T, Legal);
78 setOperationAction(ISD::OR, T, Legal);
79 setOperationAction(ISD::XOR, T, Legal);
80 setOperationAction(ISD::ADD, T, Legal);
81 setOperationAction(ISD::SUB, T, Legal);
82 setOperationAction(ISD::CTPOP, T, Legal);
83 setOperationAction(ISD::CTLZ, T, Legal);
85 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
86 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
87 setOperationAction(ISD::BSWAP, T, Legal);
90 setOperationAction(ISD::CTTZ, T, Custom);
91 setOperationAction(ISD::LOAD, T, Custom);
92 setOperationAction(ISD::MUL, T, Custom);
93 setOperationAction(ISD::MULHS, T, Custom);
94 setOperationAction(ISD::MULHU, T, Custom);
95 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
96 // Make concat-vectors custom to handle concats of more than 2 vectors.
97 setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
98 setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom);
99 setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
100 setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom);
101 setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
102 setOperationAction(ISD::ANY_EXTEND, T, Custom);
103 setOperationAction(ISD::SIGN_EXTEND, T, Custom);
104 setOperationAction(ISD::ZERO_EXTEND, T, Custom);
106 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
107 // HVX only has shifts of words and halfwords.
108 setOperationAction(ISD::SRA, T, Custom);
109 setOperationAction(ISD::SHL, T, Custom);
110 setOperationAction(ISD::SRL, T, Custom);
112 // Promote all shuffles to operate on vectors of bytes.
113 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
116 setCondCodeAction(ISD::SETNE, T, Expand);
117 setCondCodeAction(ISD::SETLE, T, Expand);
118 setCondCodeAction(ISD::SETGE, T, Expand);
119 setCondCodeAction(ISD::SETLT, T, Expand);
120 setCondCodeAction(ISD::SETULE, T, Expand);
121 setCondCodeAction(ISD::SETUGE, T, Expand);
122 setCondCodeAction(ISD::SETULT, T, Expand);
125 for (MVT T : LegalW) {
126 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
127 // independent) handling of it would convert it to a load, which is
128 // not always the optimal choice.
129 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
130 // Make concat-vectors custom to handle concats of more than 2 vectors.
131 setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
133 // Custom-lower these operations for pairs. Expand them into a concat
134 // of the corresponding operations on individual vectors.
135 setOperationAction(ISD::ANY_EXTEND, T, Custom);
136 setOperationAction(ISD::SIGN_EXTEND, T, Custom);
137 setOperationAction(ISD::ZERO_EXTEND, T, Custom);
138 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom);
139 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
140 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
141 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
143 setOperationAction(ISD::LOAD, T, Custom);
144 setOperationAction(ISD::STORE, T, Custom);
145 setOperationAction(ISD::CTLZ, T, Custom);
146 setOperationAction(ISD::CTTZ, T, Custom);
147 setOperationAction(ISD::CTPOP, T, Custom);
149 setOperationAction(ISD::ADD, T, Legal);
150 setOperationAction(ISD::SUB, T, Legal);
151 setOperationAction(ISD::MUL, T, Custom);
152 setOperationAction(ISD::MULHS, T, Custom);
153 setOperationAction(ISD::MULHU, T, Custom);
154 setOperationAction(ISD::AND, T, Custom);
155 setOperationAction(ISD::OR, T, Custom);
156 setOperationAction(ISD::XOR, T, Custom);
157 setOperationAction(ISD::SETCC, T, Custom);
158 setOperationAction(ISD::VSELECT, T, Custom);
160 setOperationAction(ISD::SRA, T, Custom);
161 setOperationAction(ISD::SHL, T, Custom);
162 setOperationAction(ISD::SRL, T, Custom);
164 // Promote all shuffles to operate on vectors of bytes.
165 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
171 for (MVT T : LegalW) {
172 // Boolean types for vector pairs will overlap with the boolean
173 // types for single vectors, e.g.
174 // v64i8 -> v64i1 (single)
175 // v64i16 -> v64i1 (pair)
176 // Set these actions first, and allow the single actions to overwrite
178 MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
179 setOperationAction(ISD::SETCC, BoolW, Custom);
180 setOperationAction(ISD::AND, BoolW, Custom);
181 setOperationAction(ISD::OR, BoolW, Custom);
182 setOperationAction(ISD::XOR, BoolW, Custom);
185 for (MVT T : LegalV) {
186 MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
187 setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom);
188 setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom);
189 setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom);
190 setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom);
191 setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom);
192 setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom);
193 setOperationAction(ISD::AND, BoolV, Legal);
194 setOperationAction(ISD::OR, BoolV, Legal);
195 setOperationAction(ISD::XOR, BoolV, Legal);
200 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
201 const SDLoc &dl, SelectionDAG &DAG) const {
202 SmallVector<SDValue,4> IntOps;
203 IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
204 for (const SDValue &Op : Ops)
205 IntOps.push_back(Op);
206 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
210 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
211 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
213 MVT ElemTy = Tys.first.getVectorElementType();
214 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
215 Tys.second.getVectorNumElements());
218 HexagonTargetLowering::TypePair
219 HexagonTargetLowering::typeSplit(MVT VecTy) const {
220 assert(VecTy.isVector());
221 unsigned NumElem = VecTy.getVectorNumElements();
222 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
223 MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
224 return { HalfTy, HalfTy };
228 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
229 MVT ElemTy = VecTy.getVectorElementType();
230 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
231 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
235 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
236 MVT ElemTy = VecTy.getVectorElementType();
237 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
238 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
242 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
243 SelectionDAG &DAG) const {
244 if (ty(Vec).getVectorElementType() == ElemTy)
246 MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
247 return DAG.getBitcast(CastTy, Vec);
251 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
252 SelectionDAG &DAG) const {
253 return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
254 Ops.second, Ops.first);
257 HexagonTargetLowering::VectorPair
258 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
259 SelectionDAG &DAG) const {
260 TypePair Tys = typeSplit(ty(Vec));
261 if (Vec.getOpcode() == HexagonISD::QCAT)
262 return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
263 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
267 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
268 return Subtarget.isHVXVectorType(Ty) &&
269 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
273 HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
274 return Subtarget.isHVXVectorType(Ty) &&
275 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
279 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
280 SelectionDAG &DAG) const {
281 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
282 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
284 unsigned ElemWidth = ElemTy.getSizeInBits();
288 unsigned L = Log2_32(ElemWidth/8);
289 const SDLoc &dl(ElemIdx);
290 return DAG.getNode(ISD::SHL, dl, MVT::i32,
291 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
295 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
296 SelectionDAG &DAG) const {
297 unsigned ElemWidth = ElemTy.getSizeInBits();
298 assert(ElemWidth >= 8 && ElemWidth <= 32);
302 if (ty(Idx) != MVT::i32)
303 Idx = DAG.getBitcast(MVT::i32, Idx);
304 const SDLoc &dl(Idx);
305 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
306 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
311 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
312 SDValue Op1, ArrayRef<int> Mask,
313 SelectionDAG &DAG) const {
315 assert(OpTy == ty(Op1));
317 MVT ElemTy = OpTy.getVectorElementType();
318 if (ElemTy == MVT::i8)
319 return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
320 assert(ElemTy.getSizeInBits() >= 8);
322 MVT ResTy = tyVector(OpTy, MVT::i8);
323 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
325 SmallVector<int,128> ByteMask;
328 for (unsigned I = 0; I != ElemSize; ++I)
329 ByteMask.push_back(-1);
331 int NewM = M*ElemSize;
332 for (unsigned I = 0; I != ElemSize; ++I)
333 ByteMask.push_back(NewM+I);
336 assert(ResTy.getVectorNumElements() == ByteMask.size());
337 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
338 opCastElem(Op1, MVT::i8, DAG), ByteMask);
342 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
343 const SDLoc &dl, MVT VecTy,
344 SelectionDAG &DAG) const {
345 unsigned VecLen = Values.size();
346 MachineFunction &MF = DAG.getMachineFunction();
347 MVT ElemTy = VecTy.getVectorElementType();
348 unsigned ElemWidth = ElemTy.getSizeInBits();
349 unsigned HwLen = Subtarget.getVectorLength();
351 unsigned ElemSize = ElemWidth / 8;
352 assert(ElemSize*VecLen == HwLen);
353 SmallVector<SDValue,32> Words;
355 if (VecTy.getVectorElementType() != MVT::i32) {
356 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
357 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
358 MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
359 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
360 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
361 Words.push_back(DAG.getBitcast(MVT::i32, W));
364 Words.assign(Values.begin(), Values.end());
367 unsigned NumWords = Words.size();
368 bool IsSplat = true, IsUndef = true;
370 for (unsigned i = 0; i != NumWords && IsSplat; ++i) {
371 if (isUndef(Words[i]))
374 if (!SplatV.getNode())
376 else if (SplatV != Words[i])
380 return DAG.getUNDEF(VecTy);
382 assert(SplatV.getNode());
383 auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
384 if (IdxN && IdxN->isNullValue())
385 return getZero(dl, VecTy, DAG);
386 return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV);
389 // Delay recognizing constant vectors until here, so that we can generate
391 SmallVector<ConstantInt*, 128> Consts(VecLen);
392 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
394 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
395 (Constant**)Consts.end());
396 Constant *CV = ConstantVector::get(Tmp);
397 unsigned Align = HwLen;
398 SDValue CP = LowerConstantPool(DAG.getConstantPool(CV, VecTy, Align), DAG);
399 return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
400 MachinePointerInfo::getConstantPool(MF), Align);
403 // Construct two halves in parallel, then or them together.
404 assert(4*Words.size() == Subtarget.getVectorLength());
405 SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
406 SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
407 SDValue S = DAG.getConstant(4, dl, MVT::i32);
408 for (unsigned i = 0; i != NumWords/2; ++i) {
409 SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
411 SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
412 {HalfV1, Words[i+NumWords/2]});
413 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
414 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
417 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
418 {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
419 SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
424 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
425 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
426 MVT PredTy = ty(PredV);
427 unsigned HwLen = Subtarget.getVectorLength();
428 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
430 if (Subtarget.isHVXVectorType(PredTy, true)) {
431 // Move the vector predicate SubV to a vector register, and scale it
432 // down to match the representation (bytes per type element) that VecV
433 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
434 // in general) element and put them at the front of the resulting
435 // vector. This subvector will then be inserted into the Q2V of VecV.
436 // To avoid having an operation that generates an illegal type (short
437 // vector), generate a full size vector.
439 SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
440 SmallVector<int,128> Mask(HwLen);
441 // Scale = BitBytes(PredV) / Given BitBytes.
442 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
443 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
445 for (unsigned i = 0; i != HwLen; ++i) {
446 unsigned Num = i % Scale;
447 unsigned Off = i / Scale;
448 Mask[BlockLen*Num + Off] = i;
450 SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
453 // Fill the bytes beyond BlockLen with 0s.
454 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
455 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
456 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
457 SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
458 return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
461 // Make sure that this is a valid scalar predicate.
462 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
464 unsigned Bytes = 8 / PredTy.getVectorNumElements();
465 SmallVector<SDValue,4> Words[2];
468 auto Lo32 = [&DAG, &dl] (SDValue P) {
469 return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P);
471 auto Hi32 = [&DAG, &dl] (SDValue P) {
472 return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P);
475 SDValue W0 = isUndef(PredV)
476 ? DAG.getUNDEF(MVT::i64)
477 : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
478 Words[IdxW].push_back(Hi32(W0));
479 Words[IdxW].push_back(Lo32(W0));
481 while (Bytes < BitBytes) {
486 for (const SDValue &W : Words[IdxW ^ 1]) {
487 SDValue T = expandPredicate(W, dl, DAG);
488 Words[IdxW].push_back(Hi32(T));
489 Words[IdxW].push_back(Lo32(T));
492 for (const SDValue &W : Words[IdxW ^ 1]) {
493 Words[IdxW].push_back(W);
494 Words[IdxW].push_back(W);
500 assert(Bytes == BitBytes);
502 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
503 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
504 for (const SDValue &W : Words[IdxW]) {
505 Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
506 Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
513 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
514 const SDLoc &dl, MVT VecTy,
515 SelectionDAG &DAG) const {
516 // Construct a vector V of bytes, such that a comparison V >u 0 would
517 // produce the required vector predicate.
518 unsigned VecLen = Values.size();
519 unsigned HwLen = Subtarget.getVectorLength();
520 assert(VecLen <= HwLen || VecLen == 8*HwLen);
521 SmallVector<SDValue,128> Bytes;
522 bool AllT = true, AllF = true;
524 auto IsTrue = [] (SDValue V) {
525 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
526 return !N->isNullValue();
529 auto IsFalse = [] (SDValue V) {
530 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
531 return N->isNullValue();
535 if (VecLen <= HwLen) {
536 // In the hardware, each bit of a vector predicate corresponds to a byte
537 // of a vector register. Calculate how many bytes does a bit of VecTy
539 assert(HwLen % VecLen == 0);
540 unsigned BitBytes = HwLen / VecLen;
541 for (SDValue V : Values) {
545 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
546 : DAG.getUNDEF(MVT::i8);
547 for (unsigned B = 0; B != BitBytes; ++B)
548 Bytes.push_back(Ext);
551 // There are as many i1 values, as there are bits in a vector register.
552 // Divide the values into groups of 8 and check that each group consists
553 // of the same value (ignoring undefs).
554 for (unsigned I = 0; I != VecLen; I += 8) {
556 // Find the first non-undef value in this group.
557 for (; B != 8; ++B) {
558 if (!Values[I+B].isUndef())
561 SDValue F = Values[I+B];
565 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
566 : DAG.getUNDEF(MVT::i8);
567 Bytes.push_back(Ext);
568 // Verify that the rest of values in the group are the same as the
571 assert(Values[I+B].isUndef() || Values[I+B] == F);
576 return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
578 return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
580 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
581 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
582 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
586 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
587 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
588 MVT ElemTy = ty(VecV).getVectorElementType();
590 unsigned ElemWidth = ElemTy.getSizeInBits();
591 assert(ElemWidth >= 8 && ElemWidth <= 32);
594 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
595 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
597 if (ElemTy == MVT::i32)
600 // Have an extracted word, need to extract the smaller element out of it.
601 // 1. Extract the bits of (the original) IdxV that correspond to the index
602 // of the desired element in the 32-bit word.
603 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
604 // 2. Extract the element from the word.
605 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
606 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
610 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
611 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
612 // Implement other return types if necessary.
613 assert(ResTy == MVT::i1);
615 unsigned HwLen = Subtarget.getVectorLength();
616 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
617 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
619 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
620 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
621 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
623 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
624 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
625 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
629 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
630 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
631 MVT ElemTy = ty(VecV).getVectorElementType();
633 unsigned ElemWidth = ElemTy.getSizeInBits();
634 assert(ElemWidth >= 8 && ElemWidth <= 32);
637 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
639 MVT VecTy = ty(VecV);
640 unsigned HwLen = Subtarget.getVectorLength();
641 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
642 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
643 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
644 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
645 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
646 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
647 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
651 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
652 if (ElemTy == MVT::i32)
653 return InsertWord(VecV, ValV, ByteIdx);
655 // If this is not inserting a 32-bit word, convert it into such a thing.
656 // 1. Extract the existing word from the target vector.
657 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
658 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
659 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
662 // 2. Treating the extracted word as a 32-bit vector, insert the given
664 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
665 MVT SubVecTy = tyVector(ty(Ext), ElemTy);
666 SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
667 ValV, SubIdx, dl, ElemTy, DAG);
669 // 3. Insert the 32-bit word back into the original vector.
670 return InsertWord(VecV, Ins, ByteIdx);
674 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
675 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
676 unsigned HwLen = Subtarget.getVectorLength();
677 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
678 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
680 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
681 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
682 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
683 ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
685 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
686 return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
690 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
691 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
692 MVT VecTy = ty(VecV);
693 unsigned HwLen = Subtarget.getVectorLength();
694 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
695 MVT ElemTy = VecTy.getVectorElementType();
696 unsigned ElemWidth = ElemTy.getSizeInBits();
698 // If the source vector is a vector pair, get the single vector containing
699 // the subvector of interest. The subvector will never overlap two single
701 if (isHvxPairTy(VecTy)) {
703 if (Idx * ElemWidth >= 8*HwLen) {
704 SubIdx = Hexagon::vsub_hi;
705 Idx -= VecTy.getVectorNumElements() / 2;
707 SubIdx = Hexagon::vsub_lo;
709 VecTy = typeSplit(VecTy).first;
710 VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV);
715 // The only meaningful subvectors of a single HVX vector are those that
716 // fit in a scalar register.
717 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
719 MVT WordTy = tyVector(VecTy, MVT::i32);
720 SDValue WordVec = DAG.getBitcast(WordTy, VecV);
721 unsigned WordIdx = (Idx*ElemWidth) / 32;
723 SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
724 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
725 if (ResTy.getSizeInBits() == 32)
726 return DAG.getBitcast(ResTy, W0);
728 SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
729 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
730 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
731 return DAG.getBitcast(ResTy, WW);
735 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
736 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
737 MVT VecTy = ty(VecV);
738 unsigned HwLen = Subtarget.getVectorLength();
739 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
740 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
741 // IdxV is required to be a constant.
742 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
744 unsigned ResLen = ResTy.getVectorNumElements();
745 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
746 unsigned Offset = Idx * BitBytes;
747 SDValue Undef = DAG.getUNDEF(ByteTy);
748 SmallVector<int,128> Mask;
750 if (Subtarget.isHVXVectorType(ResTy, true)) {
751 // Converting between two vector predicates. Since the result is shorter
752 // than the source, it will correspond to a vector predicate with the
753 // relevant bits replicated. The replication count is the ratio of the
754 // source and target vector lengths.
755 unsigned Rep = VecTy.getVectorNumElements() / ResLen;
756 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
757 for (unsigned i = 0; i != HwLen/Rep; ++i) {
758 for (unsigned j = 0; j != Rep; ++j)
759 Mask.push_back(i + Offset);
761 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
762 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
765 // Converting between a vector predicate and a scalar predicate. In the
766 // vector predicate, a group of BitBytes bits will correspond to a single
767 // i1 element of the source vector type. Those bits will all have the same
768 // value. The same will be true for ByteVec, where each byte corresponds
769 // to a bit in the vector predicate.
770 // The algorithm is to traverse the ByteVec, going over the i1 values from
771 // the source vector, and generate the corresponding representation in an
772 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
773 // elements so that the interesting 8 bytes will be in the low end of the
775 unsigned Rep = 8 / ResLen;
776 // Make sure the output fill the entire vector register, so repeat the
777 // 8-byte groups as many times as necessary.
778 for (unsigned r = 0; r != HwLen/ResLen; ++r) {
779 // This will generate the indexes of the 8 interesting bytes.
780 for (unsigned i = 0; i != ResLen; ++i) {
781 for (unsigned j = 0; j != Rep; ++j)
782 Mask.push_back(Offset + i*BitBytes);
786 SDValue Zero = getZero(dl, MVT::i32, DAG);
787 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
788 // Combine the two low words from ShuffV into a v8i8, and byte-compare
790 SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
791 SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
792 {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
793 SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
794 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
795 {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
799 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
800 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
801 MVT VecTy = ty(VecV);
802 MVT SubTy = ty(SubV);
803 unsigned HwLen = Subtarget.getVectorLength();
804 MVT ElemTy = VecTy.getVectorElementType();
805 unsigned ElemWidth = ElemTy.getSizeInBits();
807 bool IsPair = isHvxPairTy(VecTy);
808 MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
809 // The two single vectors that VecV consists of, if it's a pair.
811 SDValue SingleV = VecV;
815 V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV);
816 V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV);
818 SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
820 PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
821 if (isHvxSingleTy(SubTy)) {
822 if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
823 unsigned Idx = CN->getZExtValue();
824 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
825 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
826 return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
828 // If IdxV is not a constant, generate the two variants: with the
829 // SubV as the high and as the low subregister, and select the right
830 // pair based on the IdxV.
831 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
832 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
833 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
835 // The subvector being inserted must be entirely contained in one of
836 // the vectors V0 or V1. Set SingleV to the correct one, and update
837 // IdxV to be the index relative to the beginning of that vector.
838 SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
839 IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
840 SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
843 // The only meaningful subvectors of a single HVX vector are those that
844 // fit in a scalar register.
845 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
846 // Convert IdxV to be index in bytes.
847 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
848 if (!IdxN || !IdxN->isNullValue()) {
849 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
850 DAG.getConstant(ElemWidth/8, dl, MVT::i32));
851 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
853 // When inserting a single word, the rotation back to the original position
854 // would be by HwLen-Idx, but if two words are inserted, it will need to be
856 unsigned RolBase = HwLen;
857 if (VecTy.getSizeInBits() == 32) {
858 SDValue V = DAG.getBitcast(MVT::i32, SubV);
859 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V);
861 SDValue V = DAG.getBitcast(MVT::i64, SubV);
862 SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
863 SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
864 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
865 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
866 DAG.getConstant(4, dl, MVT::i32));
867 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
870 // If the vector wasn't ror'ed, don't ror it back.
871 if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) {
872 SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
873 DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
874 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
878 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
879 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
880 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
886 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
887 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
888 MVT VecTy = ty(VecV);
889 MVT SubTy = ty(SubV);
890 assert(Subtarget.isHVXVectorType(VecTy, true));
891 // VecV is an HVX vector predicate. SubV may be either an HVX vector
892 // predicate as well, or it can be a scalar predicate.
894 unsigned VecLen = VecTy.getVectorNumElements();
895 unsigned HwLen = Subtarget.getVectorLength();
896 assert(HwLen % VecLen == 0 && "Unexpected vector type");
898 unsigned Scale = VecLen / SubTy.getVectorNumElements();
899 unsigned BitBytes = HwLen / VecLen;
900 unsigned BlockLen = HwLen / Scale;
902 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
903 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
904 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
907 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
908 if (!IdxN || !IdxN->isNullValue()) {
909 ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
910 DAG.getConstant(BitBytes, dl, MVT::i32));
911 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
914 // ByteVec is the target vector VecV rotated in such a way that the
915 // subvector should be inserted at index 0. Generate a predicate mask
916 // and use vmux to do the insertion.
917 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
918 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
919 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
920 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
921 // Rotate ByteVec back, and convert to a vector predicate.
922 if (!IdxN || !IdxN->isNullValue()) {
923 SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
924 SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
925 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
927 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
931 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
932 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
933 // Sign- and any-extending of a vector predicate to a vector register is
934 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
935 // a vector of 1s (where the 1s are of type matching the vector type).
936 assert(Subtarget.isHVXVectorType(ResTy));
938 return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
940 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
941 SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
942 DAG.getConstant(1, dl, MVT::i32));
943 SDValue False = getZero(dl, ResTy, DAG);
944 return DAG.getSelect(dl, ResTy, VecV, True, False);
948 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
953 unsigned Size = Op.getNumOperands();
954 SmallVector<SDValue,128> Ops;
955 for (unsigned i = 0; i != Size; ++i)
956 Ops.push_back(Op.getOperand(i));
958 if (VecTy.getVectorElementType() == MVT::i1)
959 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
961 if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
962 ArrayRef<SDValue> A(Ops);
963 MVT SingleTy = typeSplit(VecTy).first;
964 SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
965 SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
966 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
969 return buildHvxVectorReg(Ops, dl, VecTy, DAG);
973 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
975 // Vector concatenation of two integer (non-bool) vectors does not need
976 // special lowering. Custom-lower concats of bool vectors and expand
977 // concats of more than 2 vectors.
980 unsigned NumOp = Op.getNumOperands();
981 if (VecTy.getVectorElementType() != MVT::i1) {
984 // Expand the other cases into a build-vector.
985 SmallVector<SDValue,8> Elems;
986 for (SDValue V : Op.getNode()->ops())
987 DAG.ExtractVectorElements(V, Elems);
988 // A vector of i16 will be broken up into a build_vector of i16's.
989 // This is a problem, since at the time of operation legalization,
990 // all operations are expected to be type-legalized, and i16 is not
991 // a legal type. If any of the extracted elements is not of a valid
992 // type, sign-extend it to a valid one.
993 for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
994 SDValue V = Elems[i];
996 if (!isTypeLegal(Ty)) {
997 EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty);
998 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
999 Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1000 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1001 V.getOperand(0), V.getOperand(1)),
1002 DAG.getValueType(Ty));
1005 // A few less complicated cases.
1006 if (V.getOpcode() == ISD::Constant)
1007 Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1008 else if (V.isUndef())
1009 Elems[i] = DAG.getUNDEF(NTy);
1011 llvm_unreachable("Unexpected vector element");
1014 return DAG.getBuildVector(VecTy, dl, Elems);
1017 assert(VecTy.getVectorElementType() == MVT::i1);
1018 unsigned HwLen = Subtarget.getVectorLength();
1019 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1021 SDValue Op0 = Op.getOperand(0);
1023 // If the operands are HVX types (i.e. not scalar predicates), then
1024 // defer the concatenation, and create QCAT instead.
1025 if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1027 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1029 ArrayRef<SDUse> U(Op.getNode()->ops());
1030 SmallVector<SDValue,4> SV(U.begin(), U.end());
1031 ArrayRef<SDValue> Ops(SV);
1033 MVT HalfTy = typeSplit(VecTy).first;
1034 SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1035 Ops.take_front(NumOp/2));
1036 SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1037 Ops.take_back(NumOp/2));
1038 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1041 // Count how many bytes (in a vector register) each bit in VecTy
1043 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1045 SmallVector<SDValue,8> Prefixes;
1046 for (SDValue V : Op.getNode()->op_values()) {
1047 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1048 Prefixes.push_back(P);
1051 unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1052 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1053 SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1054 SDValue Res = getZero(dl, ByteTy, DAG);
1055 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1056 Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1057 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1059 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1063 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1065 // Change the type of the extracted element to i32.
1066 SDValue VecV = Op.getOperand(0);
1067 MVT ElemTy = ty(VecV).getVectorElementType();
1068 const SDLoc &dl(Op);
1069 SDValue IdxV = Op.getOperand(1);
1070 if (ElemTy == MVT::i1)
1071 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1073 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1077 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1079 const SDLoc &dl(Op);
1080 SDValue VecV = Op.getOperand(0);
1081 SDValue ValV = Op.getOperand(1);
1082 SDValue IdxV = Op.getOperand(2);
1083 MVT ElemTy = ty(VecV).getVectorElementType();
1084 if (ElemTy == MVT::i1)
1085 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1087 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1091 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1093 SDValue SrcV = Op.getOperand(0);
1094 MVT SrcTy = ty(SrcV);
1096 SDValue IdxV = Op.getOperand(1);
1097 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
1098 assert(Idx % DstTy.getVectorNumElements() == 0);
1100 const SDLoc &dl(Op);
1102 MVT ElemTy = SrcTy.getVectorElementType();
1103 if (ElemTy == MVT::i1)
1104 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1106 return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1110 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1112 // Idx does not need to be a constant.
1113 SDValue VecV = Op.getOperand(0);
1114 SDValue ValV = Op.getOperand(1);
1115 SDValue IdxV = Op.getOperand(2);
1117 const SDLoc &dl(Op);
1118 MVT VecTy = ty(VecV);
1119 MVT ElemTy = VecTy.getVectorElementType();
1120 if (ElemTy == MVT::i1)
1121 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1123 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1127 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1128 // Lower any-extends of boolean vectors to sign-extends, since they
1129 // translate directly to Q2V. Zero-extending could also be done equally
1130 // fast, but Q2V is used/recognized in more places.
1131 // For all other vectors, use zero-extend.
1133 SDValue InpV = Op.getOperand(0);
1134 MVT ElemTy = ty(InpV).getVectorElementType();
1135 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1136 return LowerHvxSignExt(Op, DAG);
1137 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
1141 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1143 SDValue InpV = Op.getOperand(0);
1144 MVT ElemTy = ty(InpV).getVectorElementType();
1145 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1146 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1151 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1153 SDValue InpV = Op.getOperand(0);
1154 MVT ElemTy = ty(InpV).getVectorElementType();
1155 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1156 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1161 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1162 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
1163 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1164 const SDLoc &dl(Op);
1166 SDValue InpV = Op.getOperand(0);
1167 assert(ResTy == ty(InpV));
1169 // Calculate the vectors of 1 and bitwidth(x).
1170 MVT ElemTy = ty(InpV).getVectorElementType();
1171 unsigned ElemWidth = ElemTy.getSizeInBits();
1172 // Using uint64_t because a shift by 32 can happen.
1173 uint64_t Splat1 = 0, SplatW = 0;
1174 assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32);
1175 for (unsigned i = 0; i != 32/ElemWidth; ++i) {
1176 Splat1 = (Splat1 << ElemWidth) | 1;
1177 SplatW = (SplatW << ElemWidth) | ElemWidth;
1179 SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1180 DAG.getConstant(uint32_t(Splat1), dl, MVT::i32));
1181 SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1182 DAG.getConstant(uint32_t(SplatW), dl, MVT::i32));
1183 SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1184 DAG.getConstant(-1, dl, MVT::i32));
1185 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
1186 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
1187 // it separately in custom combine or selection).
1188 SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1189 {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1190 DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1191 return DAG.getNode(ISD::SUB, dl, ResTy,
1192 {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1196 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1198 assert(ResTy.isVector() && isHvxSingleTy(ResTy));
1199 const SDLoc &dl(Op);
1200 SmallVector<int,256> ShuffMask;
1202 MVT ElemTy = ResTy.getVectorElementType();
1203 unsigned VecLen = ResTy.getVectorNumElements();
1204 SDValue Vs = Op.getOperand(0);
1205 SDValue Vt = Op.getOperand(1);
1207 switch (ElemTy.SimpleTy) {
1209 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1210 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1211 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1212 MVT ExtTy = typeExtElem(ResTy, 2);
1213 unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
1214 : Hexagon::V6_vmpyhv;
1215 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1217 // Discard high halves of the resulting values, collect the low halves.
1218 for (unsigned I = 0; I < VecLen; I += 2) {
1219 ShuffMask.push_back(I); // Pick even element.
1220 ShuffMask.push_back(I+VecLen); // Pick odd element.
1222 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1223 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1224 return DAG.getBitcast(ResTy, BS);
1227 // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode.
1228 // (There is also V6_vmpyhv, which behaves in an analogous way to
1230 return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG);
1232 // Use the following sequence for signed word multiply:
1233 // T0 = V6_vmpyiowh Vs, Vt
1234 // T1 = V6_vaslw T0, 16
1235 // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
1236 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1237 SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1238 SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1239 SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1250 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1252 assert(ResTy.isVector());
1253 const SDLoc &dl(Op);
1254 SmallVector<int,256> ShuffMask;
1256 MVT ElemTy = ResTy.getVectorElementType();
1257 unsigned VecLen = ResTy.getVectorNumElements();
1258 SDValue Vs = Op.getOperand(0);
1259 SDValue Vt = Op.getOperand(1);
1260 bool IsSigned = Op.getOpcode() == ISD::MULHS;
1262 if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
1263 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1264 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1265 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1266 // For i16, use V6_vmpyhv, which behaves in an analogous way to
1267 // V6_vmpybv: results Lo and Hi are products of even/odd elements
1269 MVT ExtTy = typeExtElem(ResTy, 2);
1270 unsigned MpyOpc = ElemTy == MVT::i8
1271 ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1272 : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1273 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1275 // Discard low halves of the resulting values, collect the high halves.
1276 for (unsigned I = 0; I < VecLen; I += 2) {
1277 ShuffMask.push_back(I+1); // Pick even element.
1278 ShuffMask.push_back(I+VecLen+1); // Pick odd element.
1280 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1281 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1282 return DAG.getBitcast(ResTy, BS);
1285 assert(ElemTy == MVT::i32);
1286 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1290 // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1291 // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
1292 // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1293 // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
1294 // + Lo(Vs) *us Vt] >> 32
1295 // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
1296 // anything, so it cannot produce any carry over to higher bits),
1297 // so everything in [] can be shifted by 16 without loss of precision.
1298 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
1299 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
1300 // Denote Hi(Vs) = Vs':
1301 // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
1302 // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
1303 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1305 SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1306 SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1309 SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1311 SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1313 SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1317 // Unsigned mulhw. (Would expansion using signed mulhw be better?)
1319 auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
1320 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
1322 auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
1323 return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
1326 MVT PairTy = typeJoin({ResTy, ResTy});
1327 SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1328 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
1329 // Multiply-unsigned halfwords:
1330 // LoVec = Vs.uh[2i] * Vt.uh[2i],
1331 // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
1332 SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1333 // The low halves in the LoVec of the pair can be discarded. They are
1334 // not added to anything (in the full-precision product), so they cannot
1335 // produce a carry into the higher bits.
1336 SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1337 // Swap low and high halves in Vt, and do the halfword multiplication
1338 // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
1339 SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1340 SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1341 // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
1342 // These products are words, but cannot be added directly because the
1343 // sums could overflow. Add these products, by halfwords, where each sum
1344 // of a pair of halfwords gives a word.
1345 SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1346 {LoVec(T2), HiVec(T2)}, DAG);
1347 // Add the high halfwords from the products of the low halfwords.
1348 SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1349 SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1350 SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1351 SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1356 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1357 // Sign- and zero-extends are legal.
1358 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
1359 return DAG.getZeroExtendVectorInReg(Op.getOperand(0), SDLoc(Op), ty(Op));
1363 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1364 if (SDValue S = getVectorShiftByInt(Op, DAG))
1370 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1371 assert(!Op.isMachineOpcode());
1372 SmallVector<SDValue,2> OpsL, OpsH;
1373 const SDLoc &dl(Op);
1375 auto SplitVTNode = [&DAG,this] (const VTSDNode *N) {
1376 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
1377 SDValue TV = DAG.getValueType(Ty);
1378 return std::make_pair(TV, TV);
1381 for (SDValue A : Op.getNode()->ops()) {
1382 VectorPair P = Subtarget.isHVXVectorType(ty(A), true)
1383 ? opSplit(A, dl, DAG)
1384 : std::make_pair(A, A);
1385 // Special case for type operand.
1386 if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1387 if (const auto *N = dyn_cast<const VTSDNode>(A.getNode()))
1390 OpsL.push_back(P.first);
1391 OpsH.push_back(P.second);
1395 MVT HalfTy = typeSplit(ResTy).first;
1396 SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1397 SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1398 SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1403 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1404 LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode());
1405 assert(BN->isUnindexed());
1406 MVT MemTy = BN->getMemoryVT().getSimpleVT();
1407 if (!isHvxPairTy(MemTy))
1410 const SDLoc &dl(Op);
1411 unsigned HwLen = Subtarget.getVectorLength();
1412 MVT SingleTy = typeSplit(MemTy).first;
1413 SDValue Chain = BN->getChain();
1414 SDValue Base0 = BN->getBasePtr();
1415 SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1417 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
1418 if (MachineMemOperand *MMO = BN->getMemOperand()) {
1419 MachineFunction &MF = DAG.getMachineFunction();
1420 MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen);
1421 MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen);
1424 unsigned MemOpc = BN->getOpcode();
1427 if (MemOpc == ISD::LOAD) {
1428 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1429 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1430 NewOp = DAG.getMergeValues(
1431 { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
1432 DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1433 Load0.getValue(1), Load1.getValue(1)) }, dl);
1435 assert(MemOpc == ISD::STORE);
1436 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
1437 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1438 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1439 NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
1446 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1447 unsigned Opc = Op.getOpcode();
1448 bool IsPairOp = isHvxPairTy(ty(Op)) ||
1449 llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
1450 return isHvxPairTy(ty(V));
1459 return SplitHvxMemOp(Op, DAG);
1474 case ISD::SIGN_EXTEND_INREG:
1475 return SplitHvxPairOp(Op, DAG);
1482 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
1483 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
1484 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
1485 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
1486 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
1487 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
1489 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
1490 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
1491 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
1492 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
1495 case ISD::SRL: return LowerHvxShift(Op, DAG);
1496 case ISD::MUL: return LowerHvxMul(Op, DAG);
1498 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
1499 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
1501 case ISD::INTRINSIC_VOID: return Op;
1502 // Unaligned loads will be handled by the default lowering.
1503 case ISD::LOAD: return SDValue();
1508 llvm_unreachable("Unhandled HVX operation");
1512 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
1513 // If the type of the result, or any operand type are HVX vector types,
1514 // this is an HVX operation.
1515 return Subtarget.isHVXVectorType(ty(Op), true) ||
1516 llvm::any_of(Op.getNode()->ops(),
1517 [this] (SDValue V) {
1518 return Subtarget.isHVXVectorType(ty(V), true);