1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Define several functions to decode x86 specific shuffle semantics using
11 // constants from the constant pool.
13 //===----------------------------------------------------------------------===//
15 #include "Utils/X86ShuffleDecode.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/IR/Constants.h"
19 //===----------------------------------------------------------------------===//
20 // Vector Mask Decoding
21 //===----------------------------------------------------------------------===//
25 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
27 SmallVectorImpl<uint64_t> &RawMask) {
28 // It is not an error for shuffle masks to not be a vector of
29 // MaskEltSizeInBits because the constant pool uniques constants by their
30 // bit representation.
31 // e.g. the following take up the same space in the constant pool:
32 // i128 -170141183420855150465331762880109871104
34 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
36 // <4 x i32> <i32 -2147483648, i32 -2147483648,
37 // i32 -2147483648, i32 -2147483648>
38 Type *CstTy = C->getType();
39 if (!CstTy->isVectorTy())
42 Type *CstEltTy = CstTy->getVectorElementType();
43 if (!CstEltTy->isIntegerTy())
46 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
47 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
48 unsigned NumCstElts = CstTy->getVectorNumElements();
50 assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
51 "Unaligned shuffle mask size");
53 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
54 UndefElts = APInt(NumMaskElts, 0);
55 RawMask.resize(NumMaskElts, 0);
57 // Fast path - if the constants match the mask size then copy direct.
58 if (MaskEltSizeInBits == CstEltSizeInBits) {
59 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
60 for (unsigned i = 0; i != NumMaskElts; ++i) {
61 Constant *COp = C->getAggregateElement(i);
62 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
65 if (isa<UndefValue>(COp)) {
71 auto *Elt = cast<ConstantInt>(COp);
72 RawMask[i] = Elt->getValue().getZExtValue();
77 // Extract all the undef/constant element data and pack into single bitsets.
78 APInt UndefBits(CstSizeInBits, 0);
79 APInt MaskBits(CstSizeInBits, 0);
80 for (unsigned i = 0; i != NumCstElts; ++i) {
81 Constant *COp = C->getAggregateElement(i);
82 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
85 unsigned BitOffset = i * CstEltSizeInBits;
87 if (isa<UndefValue>(COp)) {
88 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
92 MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
95 // Now extract the undef/constant bit data into the raw shuffle masks.
96 for (unsigned i = 0; i != NumMaskElts; ++i) {
97 unsigned BitOffset = i * MaskEltSizeInBits;
98 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
100 // Only treat the element as UNDEF if all bits are UNDEF, otherwise
102 if (EltUndef.isAllOnesValue()) {
108 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
109 RawMask[i] = EltBits.getZExtValue();
115 void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
116 Type *MaskTy = C->getType();
117 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
119 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
120 "Unexpected vector size.");
122 // The shuffle mask requires a byte vector.
124 SmallVector<uint64_t, 64> RawMask;
125 if (!extractConstantMask(C, 8, UndefElts, RawMask))
128 unsigned NumElts = RawMask.size();
129 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
130 "Unexpected number of vector elements.");
132 for (unsigned i = 0; i != NumElts; ++i) {
134 ShuffleMask.push_back(SM_SentinelUndef);
138 uint64_t Element = RawMask[i];
139 // If the high bit (7) of the byte is set, the element is zeroed.
140 if (Element & (1 << 7))
141 ShuffleMask.push_back(SM_SentinelZero);
143 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
144 // lane of the vector we're inside.
145 unsigned Base = i & ~0xf;
147 // Only the least significant 4 bits of the byte are used.
148 int Index = Base + (Element & 0xf);
149 ShuffleMask.push_back(Index);
154 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
155 SmallVectorImpl<int> &ShuffleMask) {
156 Type *MaskTy = C->getType();
157 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
159 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
160 "Unexpected vector size.");
161 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
163 // The shuffle mask requires elements the same size as the target.
165 SmallVector<uint64_t, 16> RawMask;
166 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
169 unsigned NumElts = RawMask.size();
170 unsigned NumEltsPerLane = 128 / ElSize;
171 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
172 "Unexpected number of vector elements.");
174 for (unsigned i = 0; i != NumElts; ++i) {
176 ShuffleMask.push_back(SM_SentinelUndef);
180 int Index = i & ~(NumEltsPerLane - 1);
181 uint64_t Element = RawMask[i];
183 Index += (Element >> 1) & 0x1;
185 Index += Element & 0x3;
187 ShuffleMask.push_back(Index);
191 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
192 SmallVectorImpl<int> &ShuffleMask) {
193 Type *MaskTy = C->getType();
194 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
196 assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size.");
198 // The shuffle mask requires elements the same size as the target.
200 SmallVector<uint64_t, 8> RawMask;
201 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
204 unsigned NumElts = RawMask.size();
205 unsigned NumEltsPerLane = 128 / ElSize;
206 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
207 "Unexpected number of vector elements.");
209 for (unsigned i = 0; i != NumElts; ++i) {
211 ShuffleMask.push_back(SM_SentinelUndef);
215 // VPERMIL2 Operation.
216 // Bits[3] - Match Bit.
217 // Bits[2:1] - (Per Lane) PD Shuffle Mask.
218 // Bits[2:0] - (Per Lane) PS Shuffle Mask.
219 uint64_t Selector = RawMask[i];
220 unsigned MatchBit = (Selector >> 3) & 0x1;
223 // 0Xb X Source selected by Selector index.
224 // 10b 0 Source selected by Selector index.
227 // 11b 1 Source selected by Selector index.
228 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
229 ShuffleMask.push_back(SM_SentinelZero);
233 int Index = i & ~(NumEltsPerLane - 1);
235 Index += (Selector >> 1) & 0x1;
237 Index += Selector & 0x3;
239 int Src = (Selector >> 2) & 0x1;
240 Index += Src * NumElts;
241 ShuffleMask.push_back(Index);
245 void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
246 assert(C->getType()->getPrimitiveSizeInBits() == 128 &&
247 "Unexpected vector size.");
249 // The shuffle mask requires a byte vector.
251 SmallVector<uint64_t, 16> RawMask;
252 if (!extractConstantMask(C, 8, UndefElts, RawMask))
255 unsigned NumElts = RawMask.size();
256 assert(NumElts == 16 && "Unexpected number of vector elements.");
258 for (unsigned i = 0; i != NumElts; ++i) {
260 ShuffleMask.push_back(SM_SentinelUndef);
265 // Bits[4:0] - Byte Index (0 - 31)
266 // Bits[7:5] - Permute Operation
268 // Permute Operation:
269 // 0 - Source byte (no logical operation).
270 // 1 - Invert source byte.
271 // 2 - Bit reverse of source byte.
272 // 3 - Bit reverse of inverted source byte.
273 // 4 - 00h (zero - fill).
274 // 5 - FFh (ones - fill).
275 // 6 - Most significant bit of source byte replicated in all bit positions.
276 // 7 - Invert most significant bit of source byte and replicate in all bit
278 uint64_t Element = RawMask[i];
279 uint64_t Index = Element & 0x1F;
280 uint64_t PermuteOp = (Element >> 5) & 0x7;
282 if (PermuteOp == 4) {
283 ShuffleMask.push_back(SM_SentinelZero);
286 if (PermuteOp != 0) {
290 ShuffleMask.push_back((int)Index);
294 void DecodeVPERMVMask(const Constant *C, unsigned ElSize,
295 SmallVectorImpl<int> &ShuffleMask) {
296 Type *MaskTy = C->getType();
297 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
299 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
300 "Unexpected vector size.");
301 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
302 "Unexpected vector element size.");
304 // The shuffle mask requires elements the same size as the target.
306 SmallVector<uint64_t, 64> RawMask;
307 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
310 unsigned NumElts = RawMask.size();
312 for (unsigned i = 0; i != NumElts; ++i) {
314 ShuffleMask.push_back(SM_SentinelUndef);
317 int Index = RawMask[i] & (NumElts - 1);
318 ShuffleMask.push_back(Index);
322 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize,
323 SmallVectorImpl<int> &ShuffleMask) {
324 Type *MaskTy = C->getType();
325 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
327 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
328 "Unexpected vector size.");
329 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
330 "Unexpected vector element size.");
332 // The shuffle mask requires elements the same size as the target.
334 SmallVector<uint64_t, 64> RawMask;
335 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
338 unsigned NumElts = RawMask.size();
340 for (unsigned i = 0; i != NumElts; ++i) {
342 ShuffleMask.push_back(SM_SentinelUndef);
345 int Index = RawMask[i] & (NumElts*2 - 1);
346 ShuffleMask.push_back(Index);