1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 #include "HexagonISelLowering.h"
11 #include "HexagonRegisterInfo.h"
12 #include "HexagonSubtarget.h"
17 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
18 const SDLoc &dl, SelectionDAG &DAG) const {
19 SmallVector<SDValue,4> IntOps;
20 IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
21 for (const SDValue &Op : Ops)
23 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
27 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
28 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
30 MVT ElemTy = Tys.first.getVectorElementType();
31 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
32 Tys.second.getVectorNumElements());
35 HexagonTargetLowering::TypePair
36 HexagonTargetLowering::typeSplit(MVT VecTy) const {
37 assert(VecTy.isVector());
38 unsigned NumElem = VecTy.getVectorNumElements();
39 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
40 MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
41 return { HalfTy, HalfTy };
45 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
46 MVT ElemTy = VecTy.getVectorElementType();
47 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
48 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
52 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
53 MVT ElemTy = VecTy.getVectorElementType();
54 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
55 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
59 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
60 SelectionDAG &DAG) const {
61 if (ty(Vec).getVectorElementType() == ElemTy)
63 MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
64 return DAG.getBitcast(CastTy, Vec);
68 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
69 SelectionDAG &DAG) const {
70 return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
71 Ops.second, Ops.first);
74 HexagonTargetLowering::VectorPair
75 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
76 SelectionDAG &DAG) const {
77 TypePair Tys = typeSplit(ty(Vec));
78 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
82 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
83 SelectionDAG &DAG) const {
84 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
85 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
87 unsigned ElemWidth = ElemTy.getSizeInBits();
91 unsigned L = Log2_32(ElemWidth/8);
92 const SDLoc &dl(ElemIdx);
93 return DAG.getNode(ISD::SHL, dl, MVT::i32,
94 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
98 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
99 SelectionDAG &DAG) const {
100 unsigned ElemWidth = ElemTy.getSizeInBits();
101 assert(ElemWidth >= 8 && ElemWidth <= 32);
105 if (ty(Idx) != MVT::i32)
106 Idx = DAG.getBitcast(MVT::i32, Idx);
107 const SDLoc &dl(Idx);
108 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
109 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
114 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
115 SDValue Op1, ArrayRef<int> Mask,
116 SelectionDAG &DAG) const {
118 assert(OpTy == ty(Op1));
120 MVT ElemTy = OpTy.getVectorElementType();
121 if (ElemTy == MVT::i8)
122 return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
123 assert(ElemTy.getSizeInBits() >= 8);
125 MVT ResTy = tyVector(OpTy, MVT::i8);
126 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
128 SmallVector<int,128> ByteMask;
131 for (unsigned I = 0; I != ElemSize; ++I)
132 ByteMask.push_back(-1);
134 int NewM = M*ElemSize;
135 for (unsigned I = 0; I != ElemSize; ++I)
136 ByteMask.push_back(NewM+I);
139 assert(ResTy.getVectorNumElements() == ByteMask.size());
140 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
141 opCastElem(Op1, MVT::i8, DAG), ByteMask);
145 HexagonTargetLowering::getVecBoolVT() const {
146 return MVT::getVectorVT(MVT::i1, 8*Subtarget.getVectorLength());
150 HexagonTargetLowering::buildHvxVectorSingle(ArrayRef<SDValue> Values,
151 const SDLoc &dl, MVT VecTy,
152 SelectionDAG &DAG) const {
153 unsigned VecLen = Values.size();
154 MachineFunction &MF = DAG.getMachineFunction();
155 MVT ElemTy = VecTy.getVectorElementType();
156 unsigned ElemWidth = ElemTy.getSizeInBits();
157 unsigned HwLen = Subtarget.getVectorLength();
159 SmallVector<ConstantInt*, 128> Consts(VecLen);
160 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
162 if (llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
163 return getZero(dl, VecTy, DAG);
165 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
166 (Constant**)Consts.end());
167 Constant *CV = ConstantVector::get(Tmp);
168 unsigned Align = HwLen;
169 SDValue CP = LowerConstantPool(DAG.getConstantPool(CV, VecTy, Align), DAG);
170 return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
171 MachinePointerInfo::getConstantPool(MF), Align);
174 unsigned ElemSize = ElemWidth / 8;
175 assert(ElemSize*VecLen == HwLen);
176 SmallVector<SDValue,32> Words;
178 if (VecTy.getVectorElementType() != MVT::i32) {
179 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
180 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
181 MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
182 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
183 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
184 Words.push_back(DAG.getBitcast(MVT::i32, W));
187 Words.assign(Values.begin(), Values.end());
190 // Construct two halves in parallel, then or them together.
191 assert(4*Words.size() == Subtarget.getVectorLength());
192 SDValue HalfV0 = getNode(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
193 SDValue HalfV1 = getNode(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
194 SDValue S = DAG.getConstant(4, dl, MVT::i32);
195 unsigned NumWords = Words.size();
196 for (unsigned i = 0; i != NumWords/2; ++i) {
197 SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
199 SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
200 {HalfV1, Words[i+NumWords/2]});
201 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
202 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
205 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
206 {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
207 SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
212 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
213 const SDLoc &dl, MVT VecTy,
214 SelectionDAG &DAG) const {
215 // Construct a vector V of bytes, such that a comparison V >u 0 would
216 // produce the required vector predicate.
217 unsigned VecLen = Values.size();
218 unsigned HwLen = Subtarget.getVectorLength();
219 assert(VecLen <= HwLen || VecLen == 8*HwLen);
220 SmallVector<SDValue,128> Bytes;
222 if (VecLen <= HwLen) {
223 // In the hardware, each bit of a vector predicate corresponds to a byte
224 // of a vector register. Calculate how many bytes does a bit of VecTy
226 assert(HwLen % VecLen == 0);
227 unsigned BitBytes = HwLen / VecLen;
228 for (SDValue V : Values) {
229 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
230 : DAG.getConstant(0, dl, MVT::i8);
231 for (unsigned B = 0; B != BitBytes; ++B)
232 Bytes.push_back(Ext);
235 // There are as many i1 values, as there are bits in a vector register.
236 // Divide the values into groups of 8 and check that each group consists
237 // of the same value (ignoring undefs).
238 for (unsigned I = 0; I != VecLen; I += 8) {
240 // Find the first non-undef value in this group.
241 for (; B != 8; ++B) {
242 if (!Values[I+B].isUndef())
245 SDValue F = Values[I+B];
246 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
247 : DAG.getConstant(0, dl, MVT::i8);
248 Bytes.push_back(Ext);
249 // Verify that the rest of values in the group are the same as the
252 assert(Values[I+B].isUndef() || Values[I+B] == F);
256 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
257 SDValue ByteVec = buildHvxVectorSingle(Bytes, dl, ByteTy, DAG);
258 SDValue Cmp = DAG.getSetCC(dl, VecTy, ByteVec, getZero(dl, ByteTy, DAG),
264 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
269 unsigned Size = Op.getNumOperands();
270 SmallVector<SDValue,128> Ops;
271 for (unsigned i = 0; i != Size; ++i)
272 Ops.push_back(Op.getOperand(i));
274 if (VecTy.getVectorElementType() == MVT::i1)
275 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
277 if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
278 ArrayRef<SDValue> A(Ops);
279 MVT SingleTy = typeSplit(VecTy).first;
280 SDValue V0 = buildHvxVectorSingle(A.take_front(Size/2), dl, SingleTy, DAG);
281 SDValue V1 = buildHvxVectorSingle(A.drop_front(Size/2), dl, SingleTy, DAG);
282 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
285 return buildHvxVectorSingle(Ops, dl, VecTy, DAG);
289 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
291 // Change the type of the extracted element to i32.
292 SDValue VecV = Op.getOperand(0);
293 MVT ElemTy = ty(VecV).getVectorElementType();
294 unsigned ElemWidth = ElemTy.getSizeInBits();
295 assert(ElemWidth >= 8 && ElemWidth <= 32);
299 SDValue IdxV = Op.getOperand(1);
300 if (ty(IdxV) != MVT::i32)
301 IdxV = DAG.getBitcast(MVT::i32, IdxV);
303 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
304 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
306 if (ElemTy == MVT::i32)
309 // Have an extracted word, need to extract the smaller element out of it.
310 // 1. Extract the bits of (the original) IdxV that correspond to the index
311 // of the desired element in the 32-bit word.
312 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
313 // 2. Extract the element from the word.
314 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
315 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
319 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
322 SDValue VecV = Op.getOperand(0);
323 SDValue ValV = Op.getOperand(1);
324 SDValue IdxV = Op.getOperand(2);
325 MVT ElemTy = ty(VecV).getVectorElementType();
326 unsigned ElemWidth = ElemTy.getSizeInBits();
327 assert(ElemWidth >= 8 && ElemWidth <= 32);
330 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
332 MVT VecTy = ty(VecV);
333 unsigned HwLen = Subtarget.getVectorLength();
334 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
335 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
336 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
337 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
338 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
339 {DAG.getConstant(HwLen/4, dl, MVT::i32), MaskV});
340 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
344 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
345 if (ElemTy == MVT::i32)
346 return InsertWord(VecV, ValV, ByteIdx);
348 // If this is not inserting a 32-bit word, convert it into such a thing.
349 // 1. Extract the existing word from the target vector.
350 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
351 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
352 SDValue Ex0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
353 {opCastElem(VecV, MVT::i32, DAG), WordIdx});
354 SDValue Ext = LowerHvxExtractElement(Ex0, DAG);
356 // 2. Treating the extracted word as a 32-bit vector, insert the given
358 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
359 MVT SubVecTy = tyVector(ty(Ext), ElemTy);
360 SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
361 ValV, SubIdx, dl, ElemTy, DAG);
363 // 3. Insert the 32-bit word back into the original vector.
364 return InsertWord(VecV, Ins, ByteIdx);
368 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
370 SDValue SrcV = Op.getOperand(0);
371 MVT SrcTy = ty(SrcV);
372 unsigned SrcElems = SrcTy.getVectorNumElements();
373 SDValue IdxV = Op.getOperand(1);
374 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
376 assert(Idx == 0 || DstTy.getVectorNumElements() % Idx == 0);
379 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, DstTy, SrcV);
380 if (Idx == SrcElems/2)
381 return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, DstTy, SrcV);
386 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
388 // Idx may be variable.
389 SDValue IdxV = Op.getOperand(2);
390 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
393 unsigned Idx = IdxN->getZExtValue();
395 SDValue DstV = Op.getOperand(0);
396 SDValue SrcV = Op.getOperand(1);
397 MVT DstTy = ty(DstV);
398 MVT SrcTy = ty(SrcV);
399 unsigned DstElems = DstTy.getVectorNumElements();
400 unsigned SrcElems = SrcTy.getVectorNumElements();
401 if (2*SrcElems != DstElems)
406 return DAG.getTargetInsertSubreg(Hexagon::vsub_lo, dl, DstTy, DstV, SrcV);
408 return DAG.getTargetInsertSubreg(Hexagon::vsub_hi, dl, DstTy, DstV, SrcV);
413 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
415 if (!ResTy.isVector())
418 SmallVector<int,256> ShuffMask;
420 MVT ElemTy = ResTy.getVectorElementType();
421 unsigned VecLen = ResTy.getVectorNumElements();
422 SDValue Vs = Op.getOperand(0);
423 SDValue Vt = Op.getOperand(1);
425 switch (ElemTy.SimpleTy) {
428 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
429 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
430 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
431 // For i16, use V6_vmpyhv, which behaves in an analogous way to
432 // V6_vmpybv: results Lo and Hi are products of even/odd elements
434 MVT ExtTy = typeExtElem(ResTy, 2);
435 unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
436 : Hexagon::V6_vmpyhv;
437 SDValue M = getNode(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
439 // Discard high halves of the resulting values, collect the low halves.
440 for (unsigned I = 0; I < VecLen; I += 2) {
441 ShuffMask.push_back(I); // Pick even element.
442 ShuffMask.push_back(I+VecLen); // Pick odd element.
444 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
445 return getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
448 // Use the following sequence for signed word multiply:
449 // T0 = V6_vmpyiowh Vs, Vt
450 // T1 = V6_vaslw T0, 16
451 // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
452 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
453 SDValue T0 = getNode(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
454 SDValue T1 = getNode(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
455 SDValue T2 = getNode(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
466 HexagonTargetLowering::LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const {
467 MVT VecTy = ty(Op.getOperand(0));
468 assert(VecTy == ty(Op.getOperand(1)));
470 SDValue Cmp = Op.getOperand(2);
471 ISD::CondCode CC = cast<CondCodeSDNode>(Cmp)->get();
472 bool Negate = false, Swap = false;
474 // HVX has instructions for SETEQ, SETGT, SETUGT. The other comparisons
475 // can be arranged as operand-swapped/negated versions of these. Since
476 // the generated code will have the original CC expressed as
477 // (negate (swap-op NewCmp)),
478 // the condition code for the NewCmp should be calculated from the original
479 // CC by applying these operations in the reverse order.
481 // This could also be done through setCondCodeAction, but for negation it
482 // uses a xor with a vector of -1s, which it obtains from BUILD_VECTOR.
483 // That is far too expensive for what can be done with a single instruction.
486 case ISD::SETNE: // !eq
487 case ISD::SETLE: // !gt
488 case ISD::SETGE: // !lt
489 case ISD::SETULE: // !ugt
490 case ISD::SETUGE: // !ult
491 CC = ISD::getSetCCInverse(CC, true);
499 case ISD::SETLT: // swap gt
500 case ISD::SETULT: // swap ugt
501 CC = ISD::getSetCCSwappedOperands(CC);
508 assert(CC == ISD::SETEQ || CC == ISD::SETGT || CC == ISD::SETUGT);
510 MVT ElemTy = VecTy.getVectorElementType();
511 unsigned ElemWidth = ElemTy.getSizeInBits();
512 assert(isPowerOf2_32(ElemWidth));
514 auto getIdx = [] (unsigned Code) {
515 static const unsigned Idx[] = { ISD::SETEQ, ISD::SETGT, ISD::SETUGT };
516 for (unsigned I = 0, E = array_lengthof(Idx); I != E; ++I)
519 llvm_unreachable("Unhandled CondCode");
522 static unsigned OpcTable[3][3] = {
523 // SETEQ SETGT, SETUGT
524 /* Byte */ { Hexagon::V6_veqb, Hexagon::V6_vgtb, Hexagon::V6_vgtub },
525 /* Half */ { Hexagon::V6_veqh, Hexagon::V6_vgth, Hexagon::V6_vgtuh },
526 /* Word */ { Hexagon::V6_veqw, Hexagon::V6_vgtw, Hexagon::V6_vgtuw }
529 unsigned CmpOpc = OpcTable[Log2_32(ElemWidth)-3][getIdx(CC)];
533 SDValue OpL = Swap ? Op.getOperand(1) : Op.getOperand(0);
534 SDValue OpR = Swap ? Op.getOperand(0) : Op.getOperand(1);
535 SDValue CmpV = getNode(CmpOpc, dl, ResTy, {OpL, OpR}, DAG);
536 return Negate ? getNode(Hexagon::V6_pred_not, dl, ResTy, {CmpV}, DAG)
541 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
542 // Sign- and zero-extends are legal.
543 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
544 return DAG.getZeroExtendVectorInReg(Op.getOperand(0), SDLoc(Op), ty(Op));