1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11 // stores that can be put together into vector-stores. Next, it attempts to
12 // construct vectorizable tree using the use-def chains. If a profitable tree
13 // was found, the SLP vectorizer performs vectorization on the tree.
15 // The pass is inspired by the work described in the paper:
16 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
18 //===----------------------------------------------------------------------===//
20 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/DenseSet.h"
24 #include "llvm/ADT/MapVector.h"
25 #include "llvm/ADT/None.h"
26 #include "llvm/ADT/Optional.h"
27 #include "llvm/ADT/PostOrderIterator.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SetVector.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/SmallSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/Statistic.h"
34 #include "llvm/ADT/iterator.h"
35 #include "llvm/ADT/iterator_range.h"
36 #include "llvm/Analysis/AliasAnalysis.h"
37 #include "llvm/Analysis/CodeMetrics.h"
38 #include "llvm/Analysis/DemandedBits.h"
39 #include "llvm/Analysis/GlobalsModRef.h"
40 #include "llvm/Analysis/LoopAccessAnalysis.h"
41 #include "llvm/Analysis/LoopInfo.h"
42 #include "llvm/Analysis/MemoryLocation.h"
43 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
44 #include "llvm/Analysis/ScalarEvolution.h"
45 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
46 #include "llvm/Analysis/TargetLibraryInfo.h"
47 #include "llvm/Analysis/TargetTransformInfo.h"
48 #include "llvm/Analysis/ValueTracking.h"
49 #include "llvm/Analysis/VectorUtils.h"
50 #include "llvm/IR/Attributes.h"
51 #include "llvm/IR/BasicBlock.h"
52 #include "llvm/IR/Constant.h"
53 #include "llvm/IR/Constants.h"
54 #include "llvm/IR/DataLayout.h"
55 #include "llvm/IR/DebugLoc.h"
56 #include "llvm/IR/DerivedTypes.h"
57 #include "llvm/IR/Dominators.h"
58 #include "llvm/IR/Function.h"
59 #include "llvm/IR/IRBuilder.h"
60 #include "llvm/IR/InstrTypes.h"
61 #include "llvm/IR/Instruction.h"
62 #include "llvm/IR/Instructions.h"
63 #include "llvm/IR/IntrinsicInst.h"
64 #include "llvm/IR/Intrinsics.h"
65 #include "llvm/IR/Module.h"
66 #include "llvm/IR/NoFolder.h"
67 #include "llvm/IR/Operator.h"
68 #include "llvm/IR/PassManager.h"
69 #include "llvm/IR/PatternMatch.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/Pass.h"
77 #include "llvm/Support/Casting.h"
78 #include "llvm/Support/CommandLine.h"
79 #include "llvm/Support/Compiler.h"
80 #include "llvm/Support/DOTGraphTraits.h"
81 #include "llvm/Support/Debug.h"
82 #include "llvm/Support/ErrorHandling.h"
83 #include "llvm/Support/GraphWriter.h"
84 #include "llvm/Support/KnownBits.h"
85 #include "llvm/Support/MathExtras.h"
86 #include "llvm/Support/raw_ostream.h"
87 #include "llvm/Transforms/Utils/LoopUtils.h"
88 #include "llvm/Transforms/Vectorize.h"
100 using namespace llvm;
101 using namespace llvm::PatternMatch;
102 using namespace slpvectorizer;
104 #define SV_NAME "slp-vectorizer"
105 #define DEBUG_TYPE "SLP"
107 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
115 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
118 static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
121 "Attempt to vectorize horizontal reductions feeding into a store"));
124 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
127 /// Limits the size of scheduling regions in a block.
128 /// It avoid long compile times for _very_ large blocks where vector
129 /// instructions are spread over a wide range.
130 /// This limit is way higher than needed by real-world functions.
132 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
135 static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
139 static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
143 static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
151 // Limit the number of alias checks. The limit is chosen so that
152 // it has no negative effect on the llvm benchmarks.
153 static const unsigned AliasedCheckLimit = 10;
155 // Another limit for the alias checks: The maximum distance between load/store
156 // instructions where alias checks are done.
157 // This limit is useful for very large basic blocks.
158 static const unsigned MaxMemDepDistance = 160;
160 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161 /// regions to be handled.
162 static const int MinScheduleRegionSize = 16;
164 /// Predicate for the element types that the SLP vectorizer supports.
166 /// The most important thing to filter here are types which are invalid in LLVM
167 /// vectors. We also filter target specific types which have absolutely no
168 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169 /// avoids spending time checking the cost model and realizing that they will
170 /// be inevitably scalarized.
171 static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
176 /// \returns true if all of the instructions in \p VL are in the same block or
178 static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
188 if (BB != I->getParent())
194 /// \returns True if all of the values in \p VL are constants.
195 static bool allConstant(ArrayRef<Value *> VL) {
197 if (!isa<Constant>(i))
202 /// \returns True if all of the values in \p VL are identical.
203 static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
210 /// Checks if the vector of instructions can be represented as a shuffle, like:
211 /// %x0 = extractelement <4 x i8> %x, i32 0
212 /// %x3 = extractelement <4 x i8> %x, i32 3
213 /// %y1 = extractelement <4 x i8> %y, i32 1
214 /// %y2 = extractelement <4 x i8> %y, i32 2
215 /// %x0x0 = mul i8 %x0, %x0
216 /// %x3x3 = mul i8 %x3, %x3
217 /// %y1y1 = mul i8 %y1, %y1
218 /// %y2y2 = mul i8 %y2, %y2
219 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223 /// ret <4 x i8> %ins4
224 /// can be transformed into:
225 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
227 /// %2 = mul <4 x i8> %1, %1
229 /// We convert this initially to something like:
230 /// %x0 = extractelement <4 x i8> %x, i32 0
231 /// %x3 = extractelement <4 x i8> %x, i32 3
232 /// %y1 = extractelement <4 x i8> %y, i32 1
233 /// %y2 = extractelement <4 x i8> %y, i32 2
234 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238 /// %5 = mul <4 x i8> %4, %4
239 /// %6 = extractelement <4 x i8> %5, i32 0
240 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241 /// %7 = extractelement <4 x i8> %5, i32 1
242 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243 /// %8 = extractelement <4 x i8> %5, i32 2
244 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245 /// %9 = extractelement <4 x i8> %5, i32 3
246 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247 /// ret <4 x i8> %ins4
248 /// InstCombiner transforms this into a shuffle and vector mul
249 /// TODO: Can we split off and reuse the shuffle mask detection from
250 /// TargetTransformInfo::getInstructionThroughput?
251 static Optional<TargetTransformInfo::ShuffleKind>
252 isShuffle(ArrayRef<Value *> VL) {
253 auto *EI0 = cast<ExtractElementInst>(VL[0]);
254 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
255 Value *Vec1 = nullptr;
256 Value *Vec2 = nullptr;
257 enum ShuffleMode { Unknown, Select, Permute };
258 ShuffleMode CommonShuffleMode = Unknown;
259 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
260 auto *EI = cast<ExtractElementInst>(VL[I]);
261 auto *Vec = EI->getVectorOperand();
262 // All vector operands must have the same number of vector elements.
263 if (Vec->getType()->getVectorNumElements() != Size)
265 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
268 // Undefined behavior if Idx is negative or >= Size.
269 if (Idx->getValue().uge(Size))
271 unsigned IntIdx = Idx->getValue().getZExtValue();
272 // We can extractelement from undef vector.
273 if (isa<UndefValue>(Vec))
275 // For correct shuffling we have to have at most 2 different vector operands
276 // in all extractelement instructions.
277 if (!Vec1 || Vec1 == Vec)
279 else if (!Vec2 || Vec2 == Vec)
283 if (CommonShuffleMode == Permute)
285 // If the extract index is not the same as the operation number, it is a
288 CommonShuffleMode = Permute;
291 CommonShuffleMode = Select;
293 // If we're not crossing lanes in different vectors, consider it as blending.
294 if (CommonShuffleMode == Select && Vec2)
295 return TargetTransformInfo::SK_Select;
296 // If Vec2 was never used, we have a permutation of a single vector, otherwise
297 // we have permutation of 2 vectors.
298 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
299 : TargetTransformInfo::SK_PermuteSingleSrc;
304 /// Main data required for vectorization of instructions.
305 struct InstructionsState {
306 /// The very first instruction in the list with the main opcode.
307 Value *OpValue = nullptr;
309 /// The main/alternate instruction.
310 Instruction *MainOp = nullptr;
311 Instruction *AltOp = nullptr;
313 /// The main/alternate opcodes for the list of instructions.
314 unsigned getOpcode() const {
315 return MainOp ? MainOp->getOpcode() : 0;
318 unsigned getAltOpcode() const {
319 return AltOp ? AltOp->getOpcode() : 0;
322 /// Some of the instructions in the list have alternate opcodes.
323 bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
325 bool isOpcodeOrAlt(Instruction *I) const {
326 unsigned CheckedOpcode = I->getOpcode();
327 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
330 InstructionsState() = delete;
331 InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
332 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
335 } // end anonymous namespace
337 /// Chooses the correct key for scheduling data. If \p Op has the same (or
338 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
340 static Value *isOneOf(const InstructionsState &S, Value *Op) {
341 auto *I = dyn_cast<Instruction>(Op);
342 if (I && S.isOpcodeOrAlt(I))
347 /// \returns analysis of the Instructions in \p VL described in
348 /// InstructionsState, the Opcode that we suppose the whole list
349 /// could be vectorized even if its structure is diverse.
350 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
351 unsigned BaseIndex = 0) {
352 // Make sure these are all Instructions.
353 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
354 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
356 bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
357 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
358 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
359 unsigned AltOpcode = Opcode;
360 unsigned AltIndex = BaseIndex;
362 // Check for one alternate opcode from another BinaryOperator.
363 // TODO - generalize to support all operators (types, calls etc.).
364 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
365 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
366 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
367 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
369 if (Opcode == AltOpcode) {
370 AltOpcode = InstOpcode;
374 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
375 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
376 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
378 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
380 if (Opcode == AltOpcode) {
381 AltOpcode = InstOpcode;
386 } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
388 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
391 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
392 cast<Instruction>(VL[AltIndex]));
395 /// \returns true if all of the values in \p VL have the same type or false
397 static bool allSameType(ArrayRef<Value *> VL) {
398 Type *Ty = VL[0]->getType();
399 for (int i = 1, e = VL.size(); i < e; i++)
400 if (VL[i]->getType() != Ty)
406 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
407 static Optional<unsigned> getExtractIndex(Instruction *E) {
408 unsigned Opcode = E->getOpcode();
409 assert((Opcode == Instruction::ExtractElement ||
410 Opcode == Instruction::ExtractValue) &&
411 "Expected extractelement or extractvalue instruction.");
412 if (Opcode == Instruction::ExtractElement) {
413 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
416 return CI->getZExtValue();
418 ExtractValueInst *EI = cast<ExtractValueInst>(E);
419 if (EI->getNumIndices() != 1)
421 return *EI->idx_begin();
424 /// \returns True if in-tree use also needs extract. This refers to
425 /// possible scalar operand in vectorized instruction.
426 static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
427 TargetLibraryInfo *TLI) {
428 unsigned Opcode = UserInst->getOpcode();
430 case Instruction::Load: {
431 LoadInst *LI = cast<LoadInst>(UserInst);
432 return (LI->getPointerOperand() == Scalar);
434 case Instruction::Store: {
435 StoreInst *SI = cast<StoreInst>(UserInst);
436 return (SI->getPointerOperand() == Scalar);
438 case Instruction::Call: {
439 CallInst *CI = cast<CallInst>(UserInst);
440 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
441 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
442 return (CI->getArgOperand(1) == Scalar);
451 /// \returns the AA location that is being access by the instruction.
452 static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
453 if (StoreInst *SI = dyn_cast<StoreInst>(I))
454 return MemoryLocation::get(SI);
455 if (LoadInst *LI = dyn_cast<LoadInst>(I))
456 return MemoryLocation::get(LI);
457 return MemoryLocation();
460 /// \returns True if the instruction is not a volatile or atomic load/store.
461 static bool isSimple(Instruction *I) {
462 if (LoadInst *LI = dyn_cast<LoadInst>(I))
463 return LI->isSimple();
464 if (StoreInst *SI = dyn_cast<StoreInst>(I))
465 return SI->isSimple();
466 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
467 return !MI->isVolatile();
473 namespace slpvectorizer {
475 /// Bottom Up SLP Vectorizer.
478 using ValueList = SmallVector<Value *, 8>;
479 using InstrList = SmallVector<Instruction *, 16>;
480 using ValueSet = SmallPtrSet<Value *, 16>;
481 using StoreList = SmallVector<StoreInst *, 8>;
482 using ExtraValueToDebugLocsMap =
483 MapVector<Value *, SmallVector<Instruction *, 2>>;
485 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
486 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
487 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
488 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
489 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
490 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
491 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
492 // Use the vector register size specified by the target unless overridden
493 // by a command-line option.
494 // TODO: It would be better to limit the vectorization factor based on
495 // data type rather than just register size. For example, x86 AVX has
496 // 256-bit registers, but it does not support integer operations
497 // at that width (that requires AVX2).
498 if (MaxVectorRegSizeOption.getNumOccurrences())
499 MaxVecRegSize = MaxVectorRegSizeOption;
501 MaxVecRegSize = TTI->getRegisterBitWidth(true);
503 if (MinVectorRegSizeOption.getNumOccurrences())
504 MinVecRegSize = MinVectorRegSizeOption;
506 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
509 /// Vectorize the tree that starts with the elements in \p VL.
510 /// Returns the vectorized root.
511 Value *vectorizeTree();
513 /// Vectorize the tree but with the list of externally used values \p
514 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
515 /// generated extractvalue instructions.
516 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
518 /// \returns the cost incurred by unwanted spills and fills, caused by
519 /// holding live values over call sites.
522 /// \returns the vectorization cost of the subtree that starts at \p VL.
523 /// A negative number means that this is profitable.
526 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
527 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
528 void buildTree(ArrayRef<Value *> Roots,
529 ArrayRef<Value *> UserIgnoreLst = None);
531 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
532 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
533 /// into account (anf updating it, if required) list of externally used
534 /// values stored in \p ExternallyUsedValues.
535 void buildTree(ArrayRef<Value *> Roots,
536 ExtraValueToDebugLocsMap &ExternallyUsedValues,
537 ArrayRef<Value *> UserIgnoreLst = None);
539 /// Clear the internal data structures that are created by 'buildTree'.
541 VectorizableTree.clear();
542 ScalarToTreeEntry.clear();
544 ExternalUses.clear();
545 NumOpsWantToKeepOrder.clear();
546 NumOpsWantToKeepOriginalOrder = 0;
547 for (auto &Iter : BlocksSchedules) {
548 BlockScheduling *BS = Iter.second.get();
554 unsigned getTreeSize() const { return VectorizableTree.size(); }
556 /// Perform LICM and CSE on the newly generated gather sequences.
557 void optimizeGatherSequence();
559 /// \returns The best order of instructions for vectorization.
560 Optional<ArrayRef<unsigned>> bestOrder() const {
561 auto I = std::max_element(
562 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
563 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
564 const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
565 return D1.second < D2.second;
567 if (I == NumOpsWantToKeepOrder.end() ||
568 I->getSecond() <= NumOpsWantToKeepOriginalOrder)
571 return makeArrayRef(I->getFirst());
574 /// \return The vector element size in bits to use when vectorizing the
575 /// expression tree ending at \p V. If V is a store, the size is the width of
576 /// the stored value. Otherwise, the size is the width of the largest loaded
577 /// value reaching V. This method is used by the vectorizer to calculate
578 /// vectorization factors.
579 unsigned getVectorElementSize(Value *V);
581 /// Compute the minimum type sizes required to represent the entries in a
582 /// vectorizable tree.
583 void computeMinimumValueSizes();
585 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
586 unsigned getMaxVecRegSize() const {
587 return MaxVecRegSize;
590 // \returns minimum vector register size as set by cl::opt.
591 unsigned getMinVecRegSize() const {
592 return MinVecRegSize;
595 /// Check if ArrayType or StructType is isomorphic to some VectorType.
597 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
598 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
600 /// \returns True if the VectorizableTree is both tiny and not fully
601 /// vectorizable. We do not vectorize such trees.
602 bool isTreeTinyAndNotFullyVectorizable();
604 OptimizationRemarkEmitter *getORE() { return ORE; }
609 /// Checks if all users of \p I are the part of the vectorization tree.
610 bool areAllUsersVectorized(Instruction *I) const;
612 /// \returns the cost of the vectorizable entry.
613 int getEntryCost(TreeEntry *E);
615 /// This is the recursive part of buildTree.
616 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
618 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
619 /// be vectorized to use the original vector (or aggregate "bitcast" to a
620 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
621 /// returns false, setting \p CurrentOrder to either an empty vector or a
622 /// non-identity permutation that allows to reuse extract instructions.
623 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
624 SmallVectorImpl<unsigned> &CurrentOrder) const;
626 /// Vectorize a single entry in the tree.
627 Value *vectorizeTree(TreeEntry *E);
629 /// Vectorize a single entry in the tree, starting in \p VL.
630 Value *vectorizeTree(ArrayRef<Value *> VL);
632 /// \returns the scalarization cost for this type. Scalarization in this
633 /// context means the creation of vectors from a group of scalars.
634 int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
636 /// \returns the scalarization cost for this list of values. Assuming that
637 /// this subtree gets vectorized, we may need to extract the values from the
638 /// roots. This method calculates the cost of extracting the values.
639 int getGatherCost(ArrayRef<Value *> VL);
641 /// Set the Builder insert point to one after the last instruction in
643 void setInsertPointAfterBundle(ArrayRef<Value *> VL,
644 const InstructionsState &S);
646 /// \returns a vector from a collection of scalars in \p VL.
647 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
649 /// \returns whether the VectorizableTree is fully vectorizable and will
650 /// be beneficial even the tree height is tiny.
651 bool isFullyVectorizableTinyTree();
653 /// \reorder commutative operands in alt shuffle if they result in
655 void reorderAltShuffleOperands(const InstructionsState &S,
656 ArrayRef<Value *> VL,
657 SmallVectorImpl<Value *> &Left,
658 SmallVectorImpl<Value *> &Right);
660 /// \reorder commutative operands to get better probability of
661 /// generating vectorized code.
662 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
663 SmallVectorImpl<Value *> &Left,
664 SmallVectorImpl<Value *> &Right);
666 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
668 /// \returns true if the scalars in VL are equal to this entry.
669 bool isSame(ArrayRef<Value *> VL) const {
670 if (VL.size() == Scalars.size())
671 return std::equal(VL.begin(), VL.end(), Scalars.begin());
672 return VL.size() == ReuseShuffleIndices.size() &&
674 VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
675 [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
678 /// A vector of scalars.
681 /// The Scalars are vectorized into this value. It is initialized to Null.
682 Value *VectorizedValue = nullptr;
684 /// Do we need to gather this sequence ?
685 bool NeedToGather = false;
687 /// Does this sequence require some shuffling?
688 SmallVector<unsigned, 4> ReuseShuffleIndices;
690 /// Does this entry require reordering?
691 ArrayRef<unsigned> ReorderIndices;
693 /// Points back to the VectorizableTree.
695 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
696 /// to be a pointer and needs to be able to initialize the child iterator.
697 /// Thus we need a reference back to the container to translate the indices
699 std::vector<TreeEntry> &Container;
701 /// The TreeEntry index containing the user of this entry. We can actually
702 /// have multiple users so the data structure is not truly a tree.
703 SmallVector<int, 1> UserTreeIndices;
706 /// Create a new VectorizableTree entry.
707 void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
708 ArrayRef<unsigned> ReuseShuffleIndices = None,
709 ArrayRef<unsigned> ReorderIndices = None) {
710 VectorizableTree.emplace_back(VectorizableTree);
711 int idx = VectorizableTree.size() - 1;
712 TreeEntry *Last = &VectorizableTree[idx];
713 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
714 Last->NeedToGather = !Vectorized;
715 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
716 ReuseShuffleIndices.end());
717 Last->ReorderIndices = ReorderIndices;
719 for (int i = 0, e = VL.size(); i != e; ++i) {
720 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
721 ScalarToTreeEntry[VL[i]] = idx;
724 MustGather.insert(VL.begin(), VL.end());
727 if (UserTreeIdx >= 0)
728 Last->UserTreeIndices.push_back(UserTreeIdx);
732 /// -- Vectorization State --
733 /// Holds all of the tree entries.
734 std::vector<TreeEntry> VectorizableTree;
736 TreeEntry *getTreeEntry(Value *V) {
737 auto I = ScalarToTreeEntry.find(V);
738 if (I != ScalarToTreeEntry.end())
739 return &VectorizableTree[I->second];
743 /// Maps a specific scalar to its tree entry.
744 SmallDenseMap<Value*, int> ScalarToTreeEntry;
746 /// A list of scalars that we found that we need to keep as scalars.
749 /// This POD struct describes one external user in the vectorized tree.
750 struct ExternalUser {
751 ExternalUser(Value *S, llvm::User *U, int L)
752 : Scalar(S), User(U), Lane(L) {}
754 // Which scalar in our function.
757 // Which user that uses the scalar.
760 // Which lane does the scalar belong to.
763 using UserList = SmallVector<ExternalUser, 16>;
765 /// Checks if two instructions may access the same memory.
767 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
768 /// is invariant in the calling loop.
769 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
770 Instruction *Inst2) {
771 // First check if the result is already in the cache.
772 AliasCacheKey key = std::make_pair(Inst1, Inst2);
773 Optional<bool> &result = AliasCache[key];
774 if (result.hasValue()) {
775 return result.getValue();
777 MemoryLocation Loc2 = getLocation(Inst2, AA);
779 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
780 // Do the alias check.
781 aliased = AA->alias(Loc1, Loc2);
783 // Store the result in the cache.
788 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
790 /// Cache for alias results.
791 /// TODO: consider moving this to the AliasAnalysis itself.
792 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
794 /// Removes an instruction from its block and eventually deletes it.
795 /// It's like Instruction::eraseFromParent() except that the actual deletion
796 /// is delayed until BoUpSLP is destructed.
797 /// This is required to ensure that there are no incorrect collisions in the
798 /// AliasCache, which can happen if a new instruction is allocated at the
799 /// same address as a previously deleted instruction.
800 void eraseInstruction(Instruction *I) {
801 I->removeFromParent();
802 I->dropAllReferences();
803 DeletedInstructions.emplace_back(I);
806 /// Temporary store for deleted instructions. Instructions will be deleted
807 /// eventually when the BoUpSLP is destructed.
808 SmallVector<unique_value, 8> DeletedInstructions;
810 /// A list of values that need to extracted out of the tree.
811 /// This list holds pairs of (Internal Scalar : External User). External User
812 /// can be nullptr, it means that this Internal Scalar will be used later,
813 /// after vectorization.
814 UserList ExternalUses;
816 /// Values used only by @llvm.assume calls.
817 SmallPtrSet<const Value *, 32> EphValues;
819 /// Holds all of the instructions that we gathered.
820 SetVector<Instruction *> GatherSeq;
822 /// A list of blocks that we are going to CSE.
823 SetVector<BasicBlock *> CSEBlocks;
825 /// Contains all scheduling relevant data for an instruction.
826 /// A ScheduleData either represents a single instruction or a member of an
827 /// instruction bundle (= a group of instructions which is combined into a
828 /// vector instruction).
829 struct ScheduleData {
830 // The initial value for the dependency counters. It means that the
831 // dependencies are not calculated yet.
832 enum { InvalidDeps = -1 };
834 ScheduleData() = default;
836 void init(int BlockSchedulingRegionID, Value *OpVal) {
837 FirstInBundle = this;
838 NextInBundle = nullptr;
839 NextLoadStore = nullptr;
841 SchedulingRegionID = BlockSchedulingRegionID;
842 UnscheduledDepsInBundle = UnscheduledDeps;
847 /// Returns true if the dependency information has been calculated.
848 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
850 /// Returns true for single instructions and for bundle representatives
851 /// (= the head of a bundle).
852 bool isSchedulingEntity() const { return FirstInBundle == this; }
854 /// Returns true if it represents an instruction bundle and not only a
855 /// single instruction.
856 bool isPartOfBundle() const {
857 return NextInBundle != nullptr || FirstInBundle != this;
860 /// Returns true if it is ready for scheduling, i.e. it has no more
861 /// unscheduled depending instructions/bundles.
862 bool isReady() const {
863 assert(isSchedulingEntity() &&
864 "can't consider non-scheduling entity for ready list");
865 return UnscheduledDepsInBundle == 0 && !IsScheduled;
868 /// Modifies the number of unscheduled dependencies, also updating it for
869 /// the whole bundle.
870 int incrementUnscheduledDeps(int Incr) {
871 UnscheduledDeps += Incr;
872 return FirstInBundle->UnscheduledDepsInBundle += Incr;
875 /// Sets the number of unscheduled dependencies to the number of
877 void resetUnscheduledDeps() {
878 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
881 /// Clears all dependency information.
882 void clearDependencies() {
883 Dependencies = InvalidDeps;
884 resetUnscheduledDeps();
885 MemoryDependencies.clear();
888 void dump(raw_ostream &os) const {
889 if (!isSchedulingEntity()) {
891 } else if (NextInBundle) {
893 ScheduleData *SD = NextInBundle;
895 os << ';' << *SD->Inst;
896 SD = SD->NextInBundle;
904 Instruction *Inst = nullptr;
906 /// Points to the head in an instruction bundle (and always to this for
907 /// single instructions).
908 ScheduleData *FirstInBundle = nullptr;
910 /// Single linked list of all instructions in a bundle. Null if it is a
911 /// single instruction.
912 ScheduleData *NextInBundle = nullptr;
914 /// Single linked list of all memory instructions (e.g. load, store, call)
915 /// in the block - until the end of the scheduling region.
916 ScheduleData *NextLoadStore = nullptr;
918 /// The dependent memory instructions.
919 /// This list is derived on demand in calculateDependencies().
920 SmallVector<ScheduleData *, 4> MemoryDependencies;
922 /// This ScheduleData is in the current scheduling region if this matches
923 /// the current SchedulingRegionID of BlockScheduling.
924 int SchedulingRegionID = 0;
926 /// Used for getting a "good" final ordering of instructions.
927 int SchedulingPriority = 0;
929 /// The number of dependencies. Constitutes of the number of users of the
930 /// instruction plus the number of dependent memory instructions (if any).
931 /// This value is calculated on demand.
932 /// If InvalidDeps, the number of dependencies is not calculated yet.
933 int Dependencies = InvalidDeps;
935 /// The number of dependencies minus the number of dependencies of scheduled
936 /// instructions. As soon as this is zero, the instruction/bundle gets ready
938 /// Note that this is negative as long as Dependencies is not calculated.
939 int UnscheduledDeps = InvalidDeps;
941 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
942 /// single instructions.
943 int UnscheduledDepsInBundle = InvalidDeps;
945 /// True if this instruction is scheduled (or considered as scheduled in the
947 bool IsScheduled = false;
949 /// Opcode of the current instruction in the schedule data.
950 Value *OpValue = nullptr;
954 friend inline raw_ostream &operator<<(raw_ostream &os,
955 const BoUpSLP::ScheduleData &SD) {
961 friend struct GraphTraits<BoUpSLP *>;
962 friend struct DOTGraphTraits<BoUpSLP *>;
964 /// Contains all scheduling data for a basic block.
965 struct BlockScheduling {
966 BlockScheduling(BasicBlock *BB)
967 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
971 ScheduleStart = nullptr;
972 ScheduleEnd = nullptr;
973 FirstLoadStoreInRegion = nullptr;
974 LastLoadStoreInRegion = nullptr;
976 // Reduce the maximum schedule region size by the size of the
977 // previous scheduling run.
978 ScheduleRegionSizeLimit -= ScheduleRegionSize;
979 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
980 ScheduleRegionSizeLimit = MinScheduleRegionSize;
981 ScheduleRegionSize = 0;
983 // Make a new scheduling region, i.e. all existing ScheduleData is not
984 // in the new region yet.
985 ++SchedulingRegionID;
988 ScheduleData *getScheduleData(Value *V) {
989 ScheduleData *SD = ScheduleDataMap[V];
990 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
995 ScheduleData *getScheduleData(Value *V, Value *Key) {
997 return getScheduleData(V);
998 auto I = ExtraScheduleDataMap.find(V);
999 if (I != ExtraScheduleDataMap.end()) {
1000 ScheduleData *SD = I->second[Key];
1001 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1007 bool isInSchedulingRegion(ScheduleData *SD) {
1008 return SD->SchedulingRegionID == SchedulingRegionID;
1011 /// Marks an instruction as scheduled and puts all dependent ready
1012 /// instructions into the ready-list.
1013 template <typename ReadyListType>
1014 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1015 SD->IsScheduled = true;
1016 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1018 ScheduleData *BundleMember = SD;
1019 while (BundleMember) {
1020 if (BundleMember->Inst != BundleMember->OpValue) {
1021 BundleMember = BundleMember->NextInBundle;
1024 // Handle the def-use chain dependencies.
1025 for (Use &U : BundleMember->Inst->operands()) {
1026 auto *I = dyn_cast<Instruction>(U.get());
1029 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1030 if (OpDef && OpDef->hasValidDependencies() &&
1031 OpDef->incrementUnscheduledDeps(-1) == 0) {
1032 // There are no more unscheduled dependencies after
1033 // decrementing, so we can put the dependent instruction
1034 // into the ready list.
1035 ScheduleData *DepBundle = OpDef->FirstInBundle;
1036 assert(!DepBundle->IsScheduled &&
1037 "already scheduled bundle gets ready");
1038 ReadyList.insert(DepBundle);
1040 << "SLP: gets ready (def): " << *DepBundle << "\n");
1044 // Handle the memory dependencies.
1045 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1046 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1047 // There are no more unscheduled dependencies after decrementing,
1048 // so we can put the dependent instruction into the ready list.
1049 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1050 assert(!DepBundle->IsScheduled &&
1051 "already scheduled bundle gets ready");
1052 ReadyList.insert(DepBundle);
1054 << "SLP: gets ready (mem): " << *DepBundle << "\n");
1057 BundleMember = BundleMember->NextInBundle;
1061 void doForAllOpcodes(Value *V,
1062 function_ref<void(ScheduleData *SD)> Action) {
1063 if (ScheduleData *SD = getScheduleData(V))
1065 auto I = ExtraScheduleDataMap.find(V);
1066 if (I != ExtraScheduleDataMap.end())
1067 for (auto &P : I->second)
1068 if (P.second->SchedulingRegionID == SchedulingRegionID)
1072 /// Put all instructions into the ReadyList which are ready for scheduling.
1073 template <typename ReadyListType>
1074 void initialFillReadyList(ReadyListType &ReadyList) {
1075 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1076 doForAllOpcodes(I, [&](ScheduleData *SD) {
1077 if (SD->isSchedulingEntity() && SD->isReady()) {
1078 ReadyList.insert(SD);
1080 << "SLP: initially in ready list: " << *I << "\n");
1086 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1087 /// cyclic dependencies. This is only a dry-run, no instructions are
1088 /// actually moved at this stage.
1089 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
1090 const InstructionsState &S);
1092 /// Un-bundles a group of instructions.
1093 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1095 /// Allocates schedule data chunk.
1096 ScheduleData *allocateScheduleDataChunks();
1098 /// Extends the scheduling region so that V is inside the region.
1099 /// \returns true if the region size is within the limit.
1100 bool extendSchedulingRegion(Value *V, const InstructionsState &S);
1102 /// Initialize the ScheduleData structures for new instructions in the
1103 /// scheduling region.
1104 void initScheduleData(Instruction *FromI, Instruction *ToI,
1105 ScheduleData *PrevLoadStore,
1106 ScheduleData *NextLoadStore);
1108 /// Updates the dependency information of a bundle and of all instructions/
1109 /// bundles which depend on the original bundle.
1110 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1113 /// Sets all instruction in the scheduling region to un-scheduled.
1114 void resetSchedule();
1118 /// Simple memory allocation for ScheduleData.
1119 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1121 /// The size of a ScheduleData array in ScheduleDataChunks.
1124 /// The allocator position in the current chunk, which is the last entry
1125 /// of ScheduleDataChunks.
1128 /// Attaches ScheduleData to Instruction.
1129 /// Note that the mapping survives during all vectorization iterations, i.e.
1130 /// ScheduleData structures are recycled.
1131 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1133 /// Attaches ScheduleData to Instruction with the leading key.
1134 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1135 ExtraScheduleDataMap;
1137 struct ReadyList : SmallVector<ScheduleData *, 8> {
1138 void insert(ScheduleData *SD) { push_back(SD); }
1141 /// The ready-list for scheduling (only used for the dry-run).
1142 ReadyList ReadyInsts;
1144 /// The first instruction of the scheduling region.
1145 Instruction *ScheduleStart = nullptr;
1147 /// The first instruction _after_ the scheduling region.
1148 Instruction *ScheduleEnd = nullptr;
1150 /// The first memory accessing instruction in the scheduling region
1152 ScheduleData *FirstLoadStoreInRegion = nullptr;
1154 /// The last memory accessing instruction in the scheduling region
1156 ScheduleData *LastLoadStoreInRegion = nullptr;
1158 /// The current size of the scheduling region.
1159 int ScheduleRegionSize = 0;
1161 /// The maximum size allowed for the scheduling region.
1162 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1164 /// The ID of the scheduling region. For a new vectorization iteration this
1165 /// is incremented which "removes" all ScheduleData from the region.
1166 // Make sure that the initial SchedulingRegionID is greater than the
1167 // initial SchedulingRegionID in ScheduleData (which is 0).
1168 int SchedulingRegionID = 1;
1171 /// Attaches the BlockScheduling structures to basic blocks.
1172 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1174 /// Performs the "real" scheduling. Done before vectorization is actually
1175 /// performed in a basic block.
1176 void scheduleBlock(BlockScheduling *BS);
1178 /// List of users to ignore during scheduling and that don't need extracting.
1179 ArrayRef<Value *> UserIgnoreList;
1181 using OrdersType = SmallVector<unsigned, 4>;
1182 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
1183 /// sorted SmallVectors of unsigned.
1184 struct OrdersTypeDenseMapInfo {
1185 static OrdersType getEmptyKey() {
1191 static OrdersType getTombstoneKey() {
1197 static unsigned getHashValue(const OrdersType &V) {
1198 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1201 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
1206 /// Contains orders of operations along with the number of bundles that have
1207 /// operations in this order. It stores only those orders that require
1208 /// reordering, if reordering is not required it is counted using \a
1209 /// NumOpsWantToKeepOriginalOrder.
1210 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder;
1211 /// Number of bundles that do not require reordering.
1212 unsigned NumOpsWantToKeepOriginalOrder = 0;
1214 // Analysis and block reference.
1216 ScalarEvolution *SE;
1217 TargetTransformInfo *TTI;
1218 TargetLibraryInfo *TLI;
1222 AssumptionCache *AC;
1224 const DataLayout *DL;
1225 OptimizationRemarkEmitter *ORE;
1227 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1228 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1230 /// Instruction builder to construct the vectorized tree.
1231 IRBuilder<> Builder;
1233 /// A map of scalar integer values to the smallest bit width with which they
1234 /// can legally be represented. The values map to (width, signed) pairs,
1235 /// where "width" indicates the minimum bit width and "signed" is True if the
1236 /// value must be signed-extended, rather than zero-extended, back to its
1238 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1241 } // end namespace slpvectorizer
1243 template <> struct GraphTraits<BoUpSLP *> {
1244 using TreeEntry = BoUpSLP::TreeEntry;
1246 /// NodeRef has to be a pointer per the GraphWriter.
1247 using NodeRef = TreeEntry *;
1249 /// Add the VectorizableTree to the index iterator to be able to return
1250 /// TreeEntry pointers.
1251 struct ChildIteratorType
1252 : public iterator_adaptor_base<ChildIteratorType,
1253 SmallVector<int, 1>::iterator> {
1254 std::vector<TreeEntry> &VectorizableTree;
1256 ChildIteratorType(SmallVector<int, 1>::iterator W,
1257 std::vector<TreeEntry> &VT)
1258 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1260 NodeRef operator*() { return &VectorizableTree[*I]; }
1263 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1265 static ChildIteratorType child_begin(NodeRef N) {
1266 return {N->UserTreeIndices.begin(), N->Container};
1269 static ChildIteratorType child_end(NodeRef N) {
1270 return {N->UserTreeIndices.end(), N->Container};
1273 /// For the node iterator we just need to turn the TreeEntry iterator into a
1274 /// TreeEntry* iterator so that it dereferences to NodeRef.
1275 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1277 static nodes_iterator nodes_begin(BoUpSLP *R) {
1278 return nodes_iterator(R->VectorizableTree.begin());
1281 static nodes_iterator nodes_end(BoUpSLP *R) {
1282 return nodes_iterator(R->VectorizableTree.end());
1285 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1288 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1289 using TreeEntry = BoUpSLP::TreeEntry;
1291 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1293 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1295 raw_string_ostream OS(Str);
1296 if (isSplat(Entry->Scalars)) {
1297 OS << "<splat> " << *Entry->Scalars[0];
1300 for (auto V : Entry->Scalars) {
1303 R->ExternalUses.begin(), R->ExternalUses.end(),
1304 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1311 static std::string getNodeAttributes(const TreeEntry *Entry,
1313 if (Entry->NeedToGather)
1319 } // end namespace llvm
1321 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1322 ArrayRef<Value *> UserIgnoreLst) {
1323 ExtraValueToDebugLocsMap ExternallyUsedValues;
1324 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1327 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1328 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1329 ArrayRef<Value *> UserIgnoreLst) {
1331 UserIgnoreList = UserIgnoreLst;
1332 if (!allSameType(Roots))
1334 buildTree_rec(Roots, 0, -1);
1336 // Collect the values that we need to extract from the tree.
1337 for (TreeEntry &EIdx : VectorizableTree) {
1338 TreeEntry *Entry = &EIdx;
1340 // No need to handle users of gathered values.
1341 if (Entry->NeedToGather)
1345 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1346 Value *Scalar = Entry->Scalars[Lane];
1347 int FoundLane = Lane;
1348 if (!Entry->ReuseShuffleIndices.empty()) {
1350 std::distance(Entry->ReuseShuffleIndices.begin(),
1351 llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1354 // Check if the scalar is externally used as an extra arg.
1355 auto ExtI = ExternallyUsedValues.find(Scalar);
1356 if (ExtI != ExternallyUsedValues.end()) {
1357 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
1358 << Lane << " from " << *Scalar << ".\n");
1359 ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1361 for (User *U : Scalar->users()) {
1362 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1364 Instruction *UserInst = dyn_cast<Instruction>(U);
1368 // Skip in-tree scalars that become vectors
1369 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1370 Value *UseScalar = UseEntry->Scalars[0];
1371 // Some in-tree scalars will remain as scalar in vectorized
1372 // instructions. If that is the case, the one in Lane 0 will
1374 if (UseScalar != U ||
1375 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1376 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1378 assert(!UseEntry->NeedToGather && "Bad state");
1383 // Ignore users in the user ignore list.
1384 if (is_contained(UserIgnoreList, UserInst))
1387 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "
1388 << Lane << " from " << *Scalar << ".\n");
1389 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1395 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1397 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1399 InstructionsState S = getSameOpcode(VL);
1400 if (Depth == RecursionMaxDepth) {
1401 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1402 newTreeEntry(VL, false, UserTreeIdx);
1406 // Don't handle vectors.
1407 if (S.OpValue->getType()->isVectorTy()) {
1408 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1409 newTreeEntry(VL, false, UserTreeIdx);
1413 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1414 if (SI->getValueOperand()->getType()->isVectorTy()) {
1415 LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1416 newTreeEntry(VL, false, UserTreeIdx);
1420 // If all of the operands are identical or constant we have a simple solution.
1421 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
1422 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1423 newTreeEntry(VL, false, UserTreeIdx);
1427 // We now know that this is a vector of instructions of the same type from
1430 // Don't vectorize ephemeral values.
1431 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1432 if (EphValues.count(VL[i])) {
1433 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]
1434 << ") is ephemeral.\n");
1435 newTreeEntry(VL, false, UserTreeIdx);
1440 // Check if this is a duplicate of another entry.
1441 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1442 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n");
1443 if (!E->isSame(VL)) {
1444 LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1445 newTreeEntry(VL, false, UserTreeIdx);
1448 // Record the reuse of the tree node. FIXME, currently this is only used to
1449 // properly draw the graph rather than for the actual vectorization.
1450 E->UserTreeIndices.push_back(UserTreeIdx);
1451 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue
1456 // Check that none of the instructions in the bundle are already in the tree.
1457 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1458 auto *I = dyn_cast<Instruction>(VL[i]);
1461 if (getTreeEntry(I)) {
1462 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]
1463 << ") is already in tree.\n");
1464 newTreeEntry(VL, false, UserTreeIdx);
1469 // If any of the scalars is marked as a value that needs to stay scalar, then
1470 // we need to gather the scalars.
1471 // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
1472 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1473 if (MustGather.count(VL[i]) || is_contained(UserIgnoreList, VL[i])) {
1474 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1475 newTreeEntry(VL, false, UserTreeIdx);
1480 // Check that all of the users of the scalars that we want to vectorize are
1482 auto *VL0 = cast<Instruction>(S.OpValue);
1483 BasicBlock *BB = VL0->getParent();
1485 if (!DT->isReachableFromEntry(BB)) {
1486 // Don't go into unreachable blocks. They may contain instructions with
1487 // dependency cycles which confuse the final scheduling.
1488 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1489 newTreeEntry(VL, false, UserTreeIdx);
1493 // Check that every instruction appears once in this bundle.
1494 SmallVector<unsigned, 4> ReuseShuffleIndicies;
1495 SmallVector<Value *, 4> UniqueValues;
1496 DenseMap<Value *, unsigned> UniquePositions;
1497 for (Value *V : VL) {
1498 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1499 ReuseShuffleIndicies.emplace_back(Res.first->second);
1501 UniqueValues.emplace_back(V);
1503 if (UniqueValues.size() == VL.size()) {
1504 ReuseShuffleIndicies.clear();
1506 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
1507 if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1508 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1509 newTreeEntry(VL, false, UserTreeIdx);
1515 auto &BSRef = BlocksSchedules[BB];
1517 BSRef = llvm::make_unique<BlockScheduling>(BB);
1519 BlockScheduling &BS = *BSRef.get();
1521 if (!BS.tryScheduleBundle(VL, this, S)) {
1522 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1523 assert((!BS.getScheduleData(VL0) ||
1524 !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1525 "tryScheduleBundle should cancelScheduling on failure");
1526 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1529 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1531 unsigned ShuffleOrOp = S.isAltShuffle() ?
1532 (unsigned) Instruction::ShuffleVector : S.getOpcode();
1533 switch (ShuffleOrOp) {
1534 case Instruction::PHI: {
1535 PHINode *PH = dyn_cast<PHINode>(VL0);
1537 // Check for terminator values (e.g. invoke).
1538 for (unsigned j = 0; j < VL.size(); ++j)
1539 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1540 Instruction *Term = dyn_cast<Instruction>(
1541 cast<PHINode>(VL[j])->getIncomingValueForBlock(
1542 PH->getIncomingBlock(i)));
1543 if (Term && Term->isTerminator()) {
1545 << "SLP: Need to swizzle PHINodes (terminator use).\n");
1546 BS.cancelScheduling(VL, VL0);
1547 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1552 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1553 LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1555 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1557 // Prepare the operand vector.
1559 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1560 PH->getIncomingBlock(i)));
1562 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1566 case Instruction::ExtractValue:
1567 case Instruction::ExtractElement: {
1568 OrdersType CurrentOrder;
1569 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
1571 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
1572 ++NumOpsWantToKeepOriginalOrder;
1573 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1574 ReuseShuffleIndicies);
1577 if (!CurrentOrder.empty()) {
1579 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
1581 for (unsigned Idx : CurrentOrder)
1582 dbgs() << " " << Idx;
1585 // Insert new order with initial value 0, if it does not exist,
1586 // otherwise return the iterator to the existing one.
1587 auto StoredCurrentOrderAndNum =
1588 NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1589 ++StoredCurrentOrderAndNum->getSecond();
1590 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx, ReuseShuffleIndicies,
1591 StoredCurrentOrderAndNum->getFirst());
1594 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
1595 newTreeEntry(VL, /*Vectorized=*/false, UserTreeIdx, ReuseShuffleIndicies);
1596 BS.cancelScheduling(VL, VL0);
1599 case Instruction::Load: {
1600 // Check that a vectorized load would load the same memory as a scalar
1601 // load. For example, we don't want to vectorize loads that are smaller
1602 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1603 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1604 // from such a struct, we read/write packed bits disagreeing with the
1605 // unvectorized version.
1606 Type *ScalarTy = VL0->getType();
1608 if (DL->getTypeSizeInBits(ScalarTy) !=
1609 DL->getTypeAllocSizeInBits(ScalarTy)) {
1610 BS.cancelScheduling(VL, VL0);
1611 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1612 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1616 // Make sure all loads in the bundle are simple - we can't vectorize
1617 // atomic or volatile loads.
1618 SmallVector<Value *, 4> PointerOps(VL.size());
1619 auto POIter = PointerOps.begin();
1620 for (Value *V : VL) {
1621 auto *L = cast<LoadInst>(V);
1622 if (!L->isSimple()) {
1623 BS.cancelScheduling(VL, VL0);
1624 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1625 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1628 *POIter = L->getPointerOperand();
1632 OrdersType CurrentOrder;
1633 // Check the order of pointer operands.
1634 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
1637 if (CurrentOrder.empty()) {
1638 Ptr0 = PointerOps.front();
1639 PtrN = PointerOps.back();
1641 Ptr0 = PointerOps[CurrentOrder.front()];
1642 PtrN = PointerOps[CurrentOrder.back()];
1644 const SCEV *Scev0 = SE->getSCEV(Ptr0);
1645 const SCEV *ScevN = SE->getSCEV(PtrN);
1647 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
1648 uint64_t Size = DL->getTypeAllocSize(ScalarTy);
1649 // Check that the sorted loads are consecutive.
1650 if (Diff && Diff->getAPInt().getZExtValue() == (VL.size() - 1) * Size) {
1651 if (CurrentOrder.empty()) {
1652 // Original loads are consecutive and does not require reordering.
1653 ++NumOpsWantToKeepOriginalOrder;
1654 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1655 ReuseShuffleIndicies);
1656 LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1659 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1661 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1662 ReuseShuffleIndicies, I->getFirst());
1663 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
1669 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1670 BS.cancelScheduling(VL, VL0);
1671 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1674 case Instruction::ZExt:
1675 case Instruction::SExt:
1676 case Instruction::FPToUI:
1677 case Instruction::FPToSI:
1678 case Instruction::FPExt:
1679 case Instruction::PtrToInt:
1680 case Instruction::IntToPtr:
1681 case Instruction::SIToFP:
1682 case Instruction::UIToFP:
1683 case Instruction::Trunc:
1684 case Instruction::FPTrunc:
1685 case Instruction::BitCast: {
1686 Type *SrcTy = VL0->getOperand(0)->getType();
1687 for (unsigned i = 0; i < VL.size(); ++i) {
1688 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1689 if (Ty != SrcTy || !isValidElementType(Ty)) {
1690 BS.cancelScheduling(VL, VL0);
1691 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1693 << "SLP: Gathering casts with different src types.\n");
1697 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1698 LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1700 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1702 // Prepare the operand vector.
1704 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1706 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1710 case Instruction::ICmp:
1711 case Instruction::FCmp: {
1712 // Check that all of the compares have the same predicate.
1713 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1714 Type *ComparedTy = VL0->getOperand(0)->getType();
1715 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1716 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1717 if (Cmp->getPredicate() != P0 ||
1718 Cmp->getOperand(0)->getType() != ComparedTy) {
1719 BS.cancelScheduling(VL, VL0);
1720 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1722 << "SLP: Gathering cmp with different predicate.\n");
1727 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1728 LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1730 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1732 // Prepare the operand vector.
1734 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1736 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1740 case Instruction::Select:
1741 case Instruction::Add:
1742 case Instruction::FAdd:
1743 case Instruction::Sub:
1744 case Instruction::FSub:
1745 case Instruction::Mul:
1746 case Instruction::FMul:
1747 case Instruction::UDiv:
1748 case Instruction::SDiv:
1749 case Instruction::FDiv:
1750 case Instruction::URem:
1751 case Instruction::SRem:
1752 case Instruction::FRem:
1753 case Instruction::Shl:
1754 case Instruction::LShr:
1755 case Instruction::AShr:
1756 case Instruction::And:
1757 case Instruction::Or:
1758 case Instruction::Xor:
1759 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1760 LLVM_DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1762 // Sort operands of the instructions so that each side is more likely to
1763 // have the same opcode.
1764 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1765 ValueList Left, Right;
1766 reorderInputsAccordingToOpcode(S.getOpcode(), VL, Left, Right);
1767 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1768 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1772 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1774 // Prepare the operand vector.
1776 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1778 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1782 case Instruction::GetElementPtr: {
1783 // We don't combine GEPs with complicated (nested) indexing.
1784 for (unsigned j = 0; j < VL.size(); ++j) {
1785 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1786 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1787 BS.cancelScheduling(VL, VL0);
1788 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1793 // We can't combine several GEPs into one vector if they operate on
1795 Type *Ty0 = VL0->getOperand(0)->getType();
1796 for (unsigned j = 0; j < VL.size(); ++j) {
1797 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1800 << "SLP: not-vectorizable GEP (different types).\n");
1801 BS.cancelScheduling(VL, VL0);
1802 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1807 // We don't combine GEPs with non-constant indexes.
1808 for (unsigned j = 0; j < VL.size(); ++j) {
1809 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1810 if (!isa<ConstantInt>(Op)) {
1812 << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1813 BS.cancelScheduling(VL, VL0);
1814 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1819 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1820 LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1821 for (unsigned i = 0, e = 2; i < e; ++i) {
1823 // Prepare the operand vector.
1825 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1827 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1831 case Instruction::Store: {
1832 // Check if the stores are consecutive or of we need to swizzle them.
1833 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1834 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1835 BS.cancelScheduling(VL, VL0);
1836 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1837 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1841 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1842 LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1846 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1848 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1851 case Instruction::Call: {
1852 // Check if the calls are all to the same vectorizable intrinsic.
1853 CallInst *CI = cast<CallInst>(VL0);
1854 // Check if this is an Intrinsic call or something that can be
1855 // represented by an intrinsic call
1856 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1857 if (!isTriviallyVectorizable(ID)) {
1858 BS.cancelScheduling(VL, VL0);
1859 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1860 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1863 Function *Int = CI->getCalledFunction();
1864 Value *A1I = nullptr;
1865 if (hasVectorInstrinsicScalarOpd(ID, 1))
1866 A1I = CI->getArgOperand(1);
1867 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1868 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1869 if (!CI2 || CI2->getCalledFunction() != Int ||
1870 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1871 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1872 BS.cancelScheduling(VL, VL0);
1873 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1874 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1878 // ctlz,cttz and powi are special intrinsics whose second argument
1879 // should be same in order for them to be vectorized.
1880 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1881 Value *A1J = CI2->getArgOperand(1);
1883 BS.cancelScheduling(VL, VL0);
1884 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1885 LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1886 << " argument " << A1I << "!=" << A1J << "\n");
1890 // Verify that the bundle operands are identical between the two calls.
1891 if (CI->hasOperandBundles() &&
1892 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1893 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1894 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1895 BS.cancelScheduling(VL, VL0);
1896 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1897 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"
1898 << *CI << "!=" << *VL[i] << '\n');
1903 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1904 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1906 // Prepare the operand vector.
1907 for (Value *j : VL) {
1908 CallInst *CI2 = dyn_cast<CallInst>(j);
1909 Operands.push_back(CI2->getArgOperand(i));
1911 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1915 case Instruction::ShuffleVector:
1916 // If this is not an alternate sequence of opcode like add-sub
1917 // then do not vectorize this instruction.
1918 if (!S.isAltShuffle()) {
1919 BS.cancelScheduling(VL, VL0);
1920 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1921 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1924 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1925 LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1927 // Reorder operands if reordering would enable vectorization.
1928 if (isa<BinaryOperator>(VL0)) {
1929 ValueList Left, Right;
1930 reorderAltShuffleOperands(S, VL, Left, Right);
1931 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1932 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1936 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1938 // Prepare the operand vector.
1940 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1942 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1947 BS.cancelScheduling(VL, VL0);
1948 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1949 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1954 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1957 auto *ST = dyn_cast<StructType>(T);
1959 N = ST->getNumElements();
1960 EltTy = *ST->element_begin();
1962 N = cast<ArrayType>(T)->getNumElements();
1963 EltTy = cast<ArrayType>(T)->getElementType();
1965 if (!isValidElementType(EltTy))
1967 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1968 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1971 // Check that struct is homogeneous.
1972 for (const auto *Ty : ST->elements())
1979 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1980 SmallVectorImpl<unsigned> &CurrentOrder) const {
1981 Instruction *E0 = cast<Instruction>(OpValue);
1982 assert(E0->getOpcode() == Instruction::ExtractElement ||
1983 E0->getOpcode() == Instruction::ExtractValue);
1984 assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode");
1985 // Check if all of the extracts come from the same vector and from the
1987 Value *Vec = E0->getOperand(0);
1989 CurrentOrder.clear();
1991 // We have to extract from a vector/aggregate with the same number of elements.
1993 if (E0->getOpcode() == Instruction::ExtractValue) {
1994 const DataLayout &DL = E0->getModule()->getDataLayout();
1995 NElts = canMapToVector(Vec->getType(), DL);
1998 // Check if load can be rewritten as load of vector.
1999 LoadInst *LI = dyn_cast<LoadInst>(Vec);
2000 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2003 NElts = Vec->getType()->getVectorNumElements();
2006 if (NElts != VL.size())
2009 // Check that all of the indices extract from the correct offset.
2010 bool ShouldKeepOrder = true;
2011 unsigned E = VL.size();
2012 // Assign to all items the initial value E + 1 so we can check if the extract
2013 // instruction index was used already.
2014 // Also, later we can check that all the indices are used and we have a
2015 // consecutive access in the extract instructions, by checking that no
2016 // element of CurrentOrder still has value E + 1.
2017 CurrentOrder.assign(E, E + 1);
2019 for (; I < E; ++I) {
2020 auto *Inst = cast<Instruction>(VL[I]);
2021 if (Inst->getOperand(0) != Vec)
2023 Optional<unsigned> Idx = getExtractIndex(Inst);
2026 const unsigned ExtIdx = *Idx;
2028 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
2030 ShouldKeepOrder = false;
2031 CurrentOrder[ExtIdx] = I;
2033 if (CurrentOrder[I] != E + 1)
2035 CurrentOrder[I] = I;
2039 CurrentOrder.clear();
2043 return ShouldKeepOrder;
2046 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2047 return I->hasOneUse() ||
2048 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2049 return ScalarToTreeEntry.count(U) > 0;
2053 int BoUpSLP::getEntryCost(TreeEntry *E) {
2054 ArrayRef<Value*> VL = E->Scalars;
2056 Type *ScalarTy = VL[0]->getType();
2057 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2058 ScalarTy = SI->getValueOperand()->getType();
2059 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2060 ScalarTy = CI->getOperand(0)->getType();
2061 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2063 // If we have computed a smaller type for the expression, update VecTy so
2064 // that the costs will be accurate.
2065 if (MinBWs.count(VL[0]))
2066 VecTy = VectorType::get(
2067 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2069 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2070 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2071 int ReuseShuffleCost = 0;
2072 if (NeedToShuffleReuses) {
2074 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2076 if (E->NeedToGather) {
2077 if (allConstant(VL))
2080 return ReuseShuffleCost +
2081 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2083 if (getSameOpcode(VL).getOpcode() == Instruction::ExtractElement &&
2084 allSameType(VL) && allSameBlock(VL)) {
2085 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
2086 if (ShuffleKind.hasValue()) {
2087 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2088 for (auto *V : VL) {
2089 // If all users of instruction are going to be vectorized and this
2090 // instruction itself is not going to be vectorized, consider this
2091 // instruction as dead and remove its cost from the final cost of the
2093 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2094 !ScalarToTreeEntry.count(V)) {
2095 auto *IO = cast<ConstantInt>(
2096 cast<ExtractElementInst>(V)->getIndexOperand());
2097 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2098 IO->getZExtValue());
2101 return ReuseShuffleCost + Cost;
2104 return ReuseShuffleCost + getGatherCost(VL);
2106 InstructionsState S = getSameOpcode(VL);
2107 assert(S.getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2108 Instruction *VL0 = cast<Instruction>(S.OpValue);
2109 unsigned ShuffleOrOp = S.isAltShuffle() ?
2110 (unsigned) Instruction::ShuffleVector : S.getOpcode();
2111 switch (ShuffleOrOp) {
2112 case Instruction::PHI:
2115 case Instruction::ExtractValue:
2116 case Instruction::ExtractElement:
2117 if (NeedToShuffleReuses) {
2119 for (unsigned I : E->ReuseShuffleIndices) {
2120 if (ShuffleOrOp == Instruction::ExtractElement) {
2121 auto *IO = cast<ConstantInt>(
2122 cast<ExtractElementInst>(VL[I])->getIndexOperand());
2123 Idx = IO->getZExtValue();
2124 ReuseShuffleCost -= TTI->getVectorInstrCost(
2125 Instruction::ExtractElement, VecTy, Idx);
2127 ReuseShuffleCost -= TTI->getVectorInstrCost(
2128 Instruction::ExtractElement, VecTy, Idx);
2132 Idx = ReuseShuffleNumbers;
2133 for (Value *V : VL) {
2134 if (ShuffleOrOp == Instruction::ExtractElement) {
2135 auto *IO = cast<ConstantInt>(
2136 cast<ExtractElementInst>(V)->getIndexOperand());
2137 Idx = IO->getZExtValue();
2142 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2145 if (!E->NeedToGather) {
2146 int DeadCost = ReuseShuffleCost;
2147 if (!E->ReorderIndices.empty()) {
2148 // TODO: Merge this shuffle with the ReuseShuffleCost.
2149 DeadCost += TTI->getShuffleCost(
2150 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2152 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2153 Instruction *E = cast<Instruction>(VL[i]);
2154 // If all users are going to be vectorized, instruction can be
2155 // considered as dead.
2156 // The same, if have only one user, it will be vectorized for sure.
2157 if (areAllUsersVectorized(E)) {
2158 // Take credit for instruction that will become dead.
2159 if (E->hasOneUse()) {
2160 Instruction *Ext = E->user_back();
2161 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2162 all_of(Ext->users(),
2163 [](User *U) { return isa<GetElementPtrInst>(U); })) {
2164 // Use getExtractWithExtendCost() to calculate the cost of
2165 // extractelement/ext pair.
2166 DeadCost -= TTI->getExtractWithExtendCost(
2167 Ext->getOpcode(), Ext->getType(), VecTy, i);
2168 // Add back the cost of s|zext which is subtracted separately.
2169 DeadCost += TTI->getCastInstrCost(
2170 Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
2175 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2180 return ReuseShuffleCost + getGatherCost(VL);
2182 case Instruction::ZExt:
2183 case Instruction::SExt:
2184 case Instruction::FPToUI:
2185 case Instruction::FPToSI:
2186 case Instruction::FPExt:
2187 case Instruction::PtrToInt:
2188 case Instruction::IntToPtr:
2189 case Instruction::SIToFP:
2190 case Instruction::UIToFP:
2191 case Instruction::Trunc:
2192 case Instruction::FPTrunc:
2193 case Instruction::BitCast: {
2194 Type *SrcTy = VL0->getOperand(0)->getType();
2196 TTI->getCastInstrCost(S.getOpcode(), ScalarTy, SrcTy, VL0);
2197 if (NeedToShuffleReuses) {
2198 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2201 // Calculate the cost of this instruction.
2202 int ScalarCost = VL.size() * ScalarEltCost;
2204 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2206 // Check if the values are candidates to demote.
2207 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2208 VecCost = ReuseShuffleCost +
2209 TTI->getCastInstrCost(S.getOpcode(), VecTy, SrcVecTy, VL0);
2211 return VecCost - ScalarCost;
2213 case Instruction::FCmp:
2214 case Instruction::ICmp:
2215 case Instruction::Select: {
2216 // Calculate the cost of this instruction.
2217 int ScalarEltCost = TTI->getCmpSelInstrCost(S.getOpcode(), ScalarTy,
2218 Builder.getInt1Ty(), VL0);
2219 if (NeedToShuffleReuses) {
2220 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2222 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2223 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2224 int VecCost = TTI->getCmpSelInstrCost(S.getOpcode(), VecTy, MaskTy, VL0);
2225 return ReuseShuffleCost + VecCost - ScalarCost;
2227 case Instruction::Add:
2228 case Instruction::FAdd:
2229 case Instruction::Sub:
2230 case Instruction::FSub:
2231 case Instruction::Mul:
2232 case Instruction::FMul:
2233 case Instruction::UDiv:
2234 case Instruction::SDiv:
2235 case Instruction::FDiv:
2236 case Instruction::URem:
2237 case Instruction::SRem:
2238 case Instruction::FRem:
2239 case Instruction::Shl:
2240 case Instruction::LShr:
2241 case Instruction::AShr:
2242 case Instruction::And:
2243 case Instruction::Or:
2244 case Instruction::Xor: {
2245 // Certain instructions can be cheaper to vectorize if they have a
2246 // constant second vector operand.
2247 TargetTransformInfo::OperandValueKind Op1VK =
2248 TargetTransformInfo::OK_AnyValue;
2249 TargetTransformInfo::OperandValueKind Op2VK =
2250 TargetTransformInfo::OK_UniformConstantValue;
2251 TargetTransformInfo::OperandValueProperties Op1VP =
2252 TargetTransformInfo::OP_None;
2253 TargetTransformInfo::OperandValueProperties Op2VP =
2254 TargetTransformInfo::OP_PowerOf2;
2256 // If all operands are exactly the same ConstantInt then set the
2257 // operand kind to OK_UniformConstantValue.
2258 // If instead not all operands are constants, then set the operand kind
2259 // to OK_AnyValue. If all operands are constants but not the same,
2260 // then set the operand kind to OK_NonUniformConstantValue.
2261 ConstantInt *CInt0 = nullptr;
2262 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2263 const Instruction *I = cast<Instruction>(VL[i]);
2264 ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(1));
2266 Op2VK = TargetTransformInfo::OK_AnyValue;
2267 Op2VP = TargetTransformInfo::OP_None;
2270 if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
2271 !CInt->getValue().isPowerOf2())
2272 Op2VP = TargetTransformInfo::OP_None;
2278 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2281 SmallVector<const Value *, 4> Operands(VL0->operand_values());
2282 int ScalarEltCost = TTI->getArithmeticInstrCost(
2283 S.getOpcode(), ScalarTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands);
2284 if (NeedToShuffleReuses) {
2285 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2287 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2288 int VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy, Op1VK,
2289 Op2VK, Op1VP, Op2VP, Operands);
2290 return ReuseShuffleCost + VecCost - ScalarCost;
2292 case Instruction::GetElementPtr: {
2293 TargetTransformInfo::OperandValueKind Op1VK =
2294 TargetTransformInfo::OK_AnyValue;
2295 TargetTransformInfo::OperandValueKind Op2VK =
2296 TargetTransformInfo::OK_UniformConstantValue;
2299 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2300 if (NeedToShuffleReuses) {
2301 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2303 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2305 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2306 return ReuseShuffleCost + VecCost - ScalarCost;
2308 case Instruction::Load: {
2309 // Cost of wide load - cost of scalar loads.
2310 unsigned alignment = cast<LoadInst>(VL0)->getAlignment();
2312 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2313 if (NeedToShuffleReuses) {
2314 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2316 int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
2318 TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, VL0);
2319 if (!E->ReorderIndices.empty()) {
2320 // TODO: Merge this shuffle with the ReuseShuffleCost.
2321 VecLdCost += TTI->getShuffleCost(
2322 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2324 return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2326 case Instruction::Store: {
2327 // We know that we can merge the stores. Calculate the cost.
2328 unsigned alignment = cast<StoreInst>(VL0)->getAlignment();
2330 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2331 if (NeedToShuffleReuses) {
2332 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2334 int ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
2336 TTI->getMemoryOpCost(Instruction::Store, VecTy, alignment, 0, VL0);
2337 return ReuseShuffleCost + VecStCost - ScalarStCost;
2339 case Instruction::Call: {
2340 CallInst *CI = cast<CallInst>(VL0);
2341 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2343 // Calculate the cost of the scalar and vector calls.
2344 SmallVector<Type *, 4> ScalarTys;
2345 for (unsigned op = 0, opc = CI->getNumArgOperands(); op != opc; ++op)
2346 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2349 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2350 FMF = FPMO->getFastMathFlags();
2353 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2354 if (NeedToShuffleReuses) {
2355 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2357 int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
2359 SmallVector<Value *, 4> Args(CI->arg_operands());
2360 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2361 VecTy->getNumElements());
2363 LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCost
2364 << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2365 << " for " << *CI << "\n");
2367 return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2369 case Instruction::ShuffleVector: {
2370 assert(S.isAltShuffle() &&
2371 ((Instruction::isBinaryOp(S.getOpcode()) &&
2372 Instruction::isBinaryOp(S.getAltOpcode())) ||
2373 (Instruction::isCast(S.getOpcode()) &&
2374 Instruction::isCast(S.getAltOpcode()))) &&
2375 "Invalid Shuffle Vector Operand");
2377 if (NeedToShuffleReuses) {
2378 for (unsigned Idx : E->ReuseShuffleIndices) {
2379 Instruction *I = cast<Instruction>(VL[Idx]);
2380 ReuseShuffleCost -= TTI->getInstructionCost(
2381 I, TargetTransformInfo::TCK_RecipThroughput);
2383 for (Value *V : VL) {
2384 Instruction *I = cast<Instruction>(V);
2385 ReuseShuffleCost += TTI->getInstructionCost(
2386 I, TargetTransformInfo::TCK_RecipThroughput);
2389 for (Value *i : VL) {
2390 Instruction *I = cast<Instruction>(i);
2391 assert(S.isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
2392 ScalarCost += TTI->getInstructionCost(
2393 I, TargetTransformInfo::TCK_RecipThroughput);
2395 // VecCost is equal to sum of the cost of creating 2 vectors
2396 // and the cost of creating shuffle.
2398 if (Instruction::isBinaryOp(S.getOpcode())) {
2399 VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy);
2400 VecCost += TTI->getArithmeticInstrCost(S.getAltOpcode(), VecTy);
2402 Type *Src0SclTy = S.MainOp->getOperand(0)->getType();
2403 Type *Src1SclTy = S.AltOp->getOperand(0)->getType();
2404 VectorType *Src0Ty = VectorType::get(Src0SclTy, VL.size());
2405 VectorType *Src1Ty = VectorType::get(Src1SclTy, VL.size());
2406 VecCost = TTI->getCastInstrCost(S.getOpcode(), VecTy, Src0Ty);
2407 VecCost += TTI->getCastInstrCost(S.getAltOpcode(), VecTy, Src1Ty);
2409 VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
2410 return ReuseShuffleCost + VecCost - ScalarCost;
2413 llvm_unreachable("Unknown instruction");
2417 bool BoUpSLP::isFullyVectorizableTinyTree() {
2418 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "
2419 << VectorizableTree.size() << " is fully vectorizable .\n");
2421 // We only handle trees of heights 1 and 2.
2422 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2425 if (VectorizableTree.size() != 2)
2428 // Handle splat and all-constants stores.
2429 if (!VectorizableTree[0].NeedToGather &&
2430 (allConstant(VectorizableTree[1].Scalars) ||
2431 isSplat(VectorizableTree[1].Scalars)))
2434 // Gathering cost would be too much for tiny trees.
2435 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2441 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2442 // We can vectorize the tree if its size is greater than or equal to the
2443 // minimum size specified by the MinTreeSize command line option.
2444 if (VectorizableTree.size() >= MinTreeSize)
2447 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2448 // can vectorize it if we can prove it fully vectorizable.
2449 if (isFullyVectorizableTinyTree())
2452 assert(VectorizableTree.empty()
2453 ? ExternalUses.empty()
2454 : true && "We shouldn't have any external users");
2456 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2461 int BoUpSLP::getSpillCost() {
2462 // Walk from the bottom of the tree to the top, tracking which values are
2463 // live. When we see a call instruction that is not part of our tree,
2464 // query TTI to see if there is a cost to keeping values live over it
2465 // (for example, if spills and fills are required).
2466 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2469 SmallPtrSet<Instruction*, 4> LiveValues;
2470 Instruction *PrevInst = nullptr;
2472 for (const auto &N : VectorizableTree) {
2473 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2482 // Update LiveValues.
2483 LiveValues.erase(PrevInst);
2484 for (auto &J : PrevInst->operands()) {
2485 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2486 LiveValues.insert(cast<Instruction>(&*J));
2490 dbgs() << "SLP: #LV: " << LiveValues.size();
2491 for (auto *X : LiveValues)
2492 dbgs() << " " << X->getName();
2493 dbgs() << ", Looking at ";
2497 // Now find the sequence of instructions between PrevInst and Inst.
2498 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2500 PrevInst->getIterator().getReverse();
2501 while (InstIt != PrevInstIt) {
2502 if (PrevInstIt == PrevInst->getParent()->rend()) {
2503 PrevInstIt = Inst->getParent()->rbegin();
2507 // Debug informations don't impact spill cost.
2508 if ((isa<CallInst>(&*PrevInstIt) &&
2509 !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
2510 &*PrevInstIt != PrevInst) {
2511 SmallVector<Type*, 4> V;
2512 for (auto *II : LiveValues)
2513 V.push_back(VectorType::get(II->getType(), BundleWidth));
2514 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2526 int BoUpSLP::getTreeCost() {
2528 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
2529 << VectorizableTree.size() << ".\n");
2531 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2533 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
2534 TreeEntry &TE = VectorizableTree[I];
2536 // We create duplicate tree entries for gather sequences that have multiple
2537 // uses. However, we should not compute the cost of duplicate sequences.
2538 // For example, if we have a build vector (i.e., insertelement sequence)
2539 // that is used by more than one vector instruction, we only need to
2540 // compute the cost of the insertelement instructions once. The redundant
2541 // instructions will be eliminated by CSE.
2543 // We should consider not creating duplicate tree entries for gather
2544 // sequences, and instead add additional edges to the tree representing
2545 // their uses. Since such an approach results in fewer total entries,
2546 // existing heuristics based on tree size may yield different results.
2548 if (TE.NeedToGather &&
2549 std::any_of(std::next(VectorizableTree.begin(), I + 1),
2550 VectorizableTree.end(), [TE](TreeEntry &Entry) {
2551 return Entry.NeedToGather && Entry.isSame(TE.Scalars);
2555 int C = getEntryCost(&TE);
2556 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
2557 << " for bundle that starts with " << *TE.Scalars[0]
2562 SmallPtrSet<Value *, 16> ExtractCostCalculated;
2563 int ExtractCost = 0;
2564 for (ExternalUser &EU : ExternalUses) {
2565 // We only add extract cost once for the same scalar.
2566 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2569 // Uses by ephemeral values are free (because the ephemeral value will be
2570 // removed prior to code generation, and so the extraction will be
2571 // removed as well).
2572 if (EphValues.count(EU.User))
2575 // If we plan to rewrite the tree in a smaller type, we will need to sign
2576 // extend the extracted value back to the original type. Here, we account
2577 // for the extract and the added cost of the sign extend if needed.
2578 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2579 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2580 if (MinBWs.count(ScalarRoot)) {
2581 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2583 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2584 VecTy = VectorType::get(MinTy, BundleWidth);
2585 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2589 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2593 int SpillCost = getSpillCost();
2594 Cost += SpillCost + ExtractCost;
2598 raw_string_ostream OS(Str);
2599 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2600 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2601 << "SLP: Total Cost = " << Cost << ".\n";
2603 LLVM_DEBUG(dbgs() << Str);
2606 ViewGraph(this, "SLP" + F->getName(), false, Str);
2611 int BoUpSLP::getGatherCost(Type *Ty,
2612 const DenseSet<unsigned> &ShuffledIndices) {
2614 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2615 if (!ShuffledIndices.count(i))
2616 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2617 if (!ShuffledIndices.empty())
2618 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2622 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2623 // Find the type of the operands in VL.
2624 Type *ScalarTy = VL[0]->getType();
2625 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2626 ScalarTy = SI->getValueOperand()->getType();
2627 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2628 // Find the cost of inserting/extracting values from the vector.
2629 // Check if the same elements are inserted several times and count them as
2630 // shuffle candidates.
2631 DenseSet<unsigned> ShuffledElements;
2632 DenseSet<Value *> UniqueElements;
2633 // Iterate in reverse order to consider insert elements with the high cost.
2634 for (unsigned I = VL.size(); I > 0; --I) {
2635 unsigned Idx = I - 1;
2636 if (!UniqueElements.insert(VL[Idx]).second)
2637 ShuffledElements.insert(Idx);
2639 return getGatherCost(VecTy, ShuffledElements);
2642 // Reorder commutative operations in alternate shuffle if the resulting vectors
2643 // are consecutive loads. This would allow us to vectorize the tree.
2644 // If we have something like-
2645 // load a[0] - load b[0]
2646 // load b[1] + load a[1]
2647 // load a[2] - load b[2]
2648 // load a[3] + load b[3]
2649 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2651 void BoUpSLP::reorderAltShuffleOperands(const InstructionsState &S,
2652 ArrayRef<Value *> VL,
2653 SmallVectorImpl<Value *> &Left,
2654 SmallVectorImpl<Value *> &Right) {
2655 // Push left and right operands of binary operation into Left and Right
2656 for (Value *V : VL) {
2657 auto *I = cast<Instruction>(V);
2658 assert(S.isOpcodeOrAlt(I) && "Incorrect instruction in vector");
2659 Left.push_back(I->getOperand(0));
2660 Right.push_back(I->getOperand(1));
2663 // Reorder if we have a commutative operation and consecutive access
2664 // are on either side of the alternate instructions.
2665 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2666 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2667 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2668 Instruction *VL1 = cast<Instruction>(VL[j]);
2669 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2670 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2671 std::swap(Left[j], Right[j]);
2673 } else if (VL2->isCommutative() &&
2674 isConsecutiveAccess(L, L1, *DL, *SE)) {
2675 std::swap(Left[j + 1], Right[j + 1]);
2681 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2682 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2683 Instruction *VL1 = cast<Instruction>(VL[j]);
2684 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2685 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2686 std::swap(Left[j], Right[j]);
2688 } else if (VL2->isCommutative() &&
2689 isConsecutiveAccess(L, L1, *DL, *SE)) {
2690 std::swap(Left[j + 1], Right[j + 1]);
2699 // Return true if I should be commuted before adding it's left and right
2700 // operands to the arrays Left and Right.
2702 // The vectorizer is trying to either have all elements one side being
2703 // instruction with the same opcode to enable further vectorization, or having
2704 // a splat to lower the vectorizing cost.
2705 static bool shouldReorderOperands(
2706 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2707 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2708 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2709 VLeft = I.getOperand(0);
2710 VRight = I.getOperand(1);
2711 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2713 if (VRight == Right[i - 1])
2714 // Preserve SplatRight
2716 if (VLeft == Right[i - 1]) {
2717 // Commuting would preserve SplatRight, but we don't want to break
2718 // SplatLeft either, i.e. preserve the original order if possible.
2719 // (FIXME: why do we care?)
2720 if (SplatLeft && VLeft == Left[i - 1])
2725 // Symmetrically handle Right side.
2727 if (VLeft == Left[i - 1])
2728 // Preserve SplatLeft
2730 if (VRight == Left[i - 1])
2734 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2735 Instruction *IRight = dyn_cast<Instruction>(VRight);
2737 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2738 // it and not the right, in this case we want to commute.
2739 if (AllSameOpcodeRight) {
2740 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2741 if (IRight && RightPrevOpcode == IRight->getOpcode())
2742 // Do not commute, a match on the right preserves AllSameOpcodeRight
2744 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2745 // We have a match and may want to commute, but first check if there is
2746 // not also a match on the existing operands on the Left to preserve
2747 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2748 // (FIXME: why do we care?)
2749 if (AllSameOpcodeLeft && ILeft &&
2750 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2755 // Symmetrically handle Left side.
2756 if (AllSameOpcodeLeft) {
2757 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2758 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2760 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2766 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2767 ArrayRef<Value *> VL,
2768 SmallVectorImpl<Value *> &Left,
2769 SmallVectorImpl<Value *> &Right) {
2771 // Peel the first iteration out of the loop since there's nothing
2772 // interesting to do anyway and it simplifies the checks in the loop.
2773 auto *I = cast<Instruction>(VL[0]);
2774 Value *VLeft = I->getOperand(0);
2775 Value *VRight = I->getOperand(1);
2776 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2777 // Favor having instruction to the right. FIXME: why?
2778 std::swap(VLeft, VRight);
2779 Left.push_back(VLeft);
2780 Right.push_back(VRight);
2783 // Keep track if we have instructions with all the same opcode on one side.
2784 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2785 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2786 // Keep track if we have one side with all the same value (broadcast).
2787 bool SplatLeft = true;
2788 bool SplatRight = true;
2790 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2791 Instruction *I = cast<Instruction>(VL[i]);
2792 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2793 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2794 "Can only process commutative instruction");
2795 // Commute to favor either a splat or maximizing having the same opcodes on
2799 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2800 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2802 Left.push_back(VRight);
2803 Right.push_back(VLeft);
2805 Left.push_back(VLeft);
2806 Right.push_back(VRight);
2808 // Update Splat* and AllSameOpcode* after the insertion.
2809 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2810 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2811 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2812 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2813 cast<Instruction>(Left[i])->getOpcode());
2814 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2815 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2816 cast<Instruction>(Right[i])->getOpcode());
2819 // If one operand end up being broadcast, return this operand order.
2820 if (SplatRight || SplatLeft)
2823 // Finally check if we can get longer vectorizable chain by reordering
2824 // without breaking the good operand order detected above.
2825 // E.g. If we have something like-
2826 // load a[0] load b[0]
2827 // load b[1] load a[1]
2828 // load a[2] load b[2]
2829 // load a[3] load b[3]
2830 // Reordering the second load b[1] load a[1] would allow us to vectorize
2831 // this code and we still retain AllSameOpcode property.
2832 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2834 // add a[0],c[0] load b[0]
2835 // add a[1],c[2] load b[1]
2837 // add a[3],c[3] load b[3]
2838 for (unsigned j = 0, e = VL.size() - 1; j < e; ++j) {
2839 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2840 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2841 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2842 std::swap(Left[j + 1], Right[j + 1]);
2847 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2848 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2849 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2850 std::swap(Left[j + 1], Right[j + 1]);
2859 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL,
2860 const InstructionsState &S) {
2861 // Get the basic block this bundle is in. All instructions in the bundle
2862 // should be in this block.
2863 auto *Front = cast<Instruction>(S.OpValue);
2864 auto *BB = Front->getParent();
2865 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2866 auto *I = cast<Instruction>(V);
2867 return !S.isOpcodeOrAlt(I) || I->getParent() == BB;
2870 // The last instruction in the bundle in program order.
2871 Instruction *LastInst = nullptr;
2873 // Find the last instruction. The common case should be that BB has been
2874 // scheduled, and the last instruction is VL.back(). So we start with
2875 // VL.back() and iterate over schedule data until we reach the end of the
2876 // bundle. The end of the bundle is marked by null ScheduleData.
2877 if (BlocksSchedules.count(BB)) {
2879 BlocksSchedules[BB]->getScheduleData(isOneOf(S, VL.back()));
2880 if (Bundle && Bundle->isPartOfBundle())
2881 for (; Bundle; Bundle = Bundle->NextInBundle)
2882 if (Bundle->OpValue == Bundle->Inst)
2883 LastInst = Bundle->Inst;
2886 // LastInst can still be null at this point if there's either not an entry
2887 // for BB in BlocksSchedules or there's no ScheduleData available for
2888 // VL.back(). This can be the case if buildTree_rec aborts for various
2889 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2890 // size is reached, etc.). ScheduleData is initialized in the scheduling
2893 // If this happens, we can still find the last instruction by brute force. We
2894 // iterate forwards from Front (inclusive) until we either see all
2895 // instructions in the bundle or reach the end of the block. If Front is the
2896 // last instruction in program order, LastInst will be set to Front, and we
2897 // will visit all the remaining instructions in the block.
2899 // One of the reasons we exit early from buildTree_rec is to place an upper
2900 // bound on compile-time. Thus, taking an additional compile-time hit here is
2901 // not ideal. However, this should be exceedingly rare since it requires that
2902 // we both exit early from buildTree_rec and that the bundle be out-of-order
2903 // (causing us to iterate all the way to the end of the block).
2905 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2906 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2907 if (Bundle.erase(&I) && S.isOpcodeOrAlt(&I))
2914 // Set the insertion point after the last instruction in the bundle. Set the
2915 // debug location to Front.
2916 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2917 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2920 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2921 Value *Vec = UndefValue::get(Ty);
2922 // Generate the 'InsertElement' instruction.
2923 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2924 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2925 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2926 GatherSeq.insert(Insrt);
2927 CSEBlocks.insert(Insrt->getParent());
2929 // Add to our 'need-to-extract' list.
2930 if (TreeEntry *E = getTreeEntry(VL[i])) {
2931 // Find which lane we need to extract.
2933 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2934 // Is this the lane of the scalar that we are looking for ?
2935 if (E->Scalars[Lane] == VL[i]) {
2940 assert(FoundLane >= 0 && "Could not find the correct lane");
2941 if (!E->ReuseShuffleIndices.empty()) {
2943 std::distance(E->ReuseShuffleIndices.begin(),
2944 llvm::find(E->ReuseShuffleIndices, FoundLane));
2946 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2954 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2955 InstructionsState S = getSameOpcode(VL);
2956 if (S.getOpcode()) {
2957 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2958 if (E->isSame(VL)) {
2959 Value *V = vectorizeTree(E);
2960 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
2961 // We need to get the vectorized value but without shuffle.
2962 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
2963 V = SV->getOperand(0);
2965 // Reshuffle to get only unique values.
2966 SmallVector<unsigned, 4> UniqueIdxs;
2967 SmallSet<unsigned, 4> UsedIdxs;
2968 for(unsigned Idx : E->ReuseShuffleIndices)
2969 if (UsedIdxs.insert(Idx).second)
2970 UniqueIdxs.emplace_back(Idx);
2971 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
2980 Type *ScalarTy = S.OpValue->getType();
2981 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2982 ScalarTy = SI->getValueOperand()->getType();
2984 // Check that every instruction appears once in this bundle.
2985 SmallVector<unsigned, 4> ReuseShuffleIndicies;
2986 SmallVector<Value *, 4> UniqueValues;
2987 if (VL.size() > 2) {
2988 DenseMap<Value *, unsigned> UniquePositions;
2989 for (Value *V : VL) {
2990 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2991 ReuseShuffleIndicies.emplace_back(Res.first->second);
2992 if (Res.second || isa<Constant>(V))
2993 UniqueValues.emplace_back(V);
2995 // Do not shuffle single element or if number of unique values is not power
2997 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
2998 !llvm::isPowerOf2_32(UniqueValues.size()))
2999 ReuseShuffleIndicies.clear();
3003 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3005 Value *V = Gather(VL, VecTy);
3006 if (!ReuseShuffleIndicies.empty()) {
3007 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3008 ReuseShuffleIndicies, "shuffle");
3009 if (auto *I = dyn_cast<Instruction>(V)) {
3010 GatherSeq.insert(I);
3011 CSEBlocks.insert(I->getParent());
3017 static void inversePermutation(ArrayRef<unsigned> Indices,
3018 SmallVectorImpl<unsigned> &Mask) {
3020 const unsigned E = Indices.size();
3022 for (unsigned I = 0; I < E; ++I)
3023 Mask[Indices[I]] = I;
3026 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
3027 IRBuilder<>::InsertPointGuard Guard(Builder);
3029 if (E->VectorizedValue) {
3030 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
3031 return E->VectorizedValue;
3034 InstructionsState S = getSameOpcode(E->Scalars);
3035 Instruction *VL0 = cast<Instruction>(S.OpValue);
3036 Type *ScalarTy = VL0->getType();
3037 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3038 ScalarTy = SI->getValueOperand()->getType();
3039 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
3041 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3043 if (E->NeedToGather) {
3044 setInsertPointAfterBundle(E->Scalars, S);
3045 auto *V = Gather(E->Scalars, VecTy);
3046 if (NeedToShuffleReuses) {
3047 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3048 E->ReuseShuffleIndices, "shuffle");
3049 if (auto *I = dyn_cast<Instruction>(V)) {
3050 GatherSeq.insert(I);
3051 CSEBlocks.insert(I->getParent());
3054 E->VectorizedValue = V;
3058 unsigned ShuffleOrOp = S.isAltShuffle() ?
3059 (unsigned) Instruction::ShuffleVector : S.getOpcode();
3060 switch (ShuffleOrOp) {
3061 case Instruction::PHI: {
3062 PHINode *PH = dyn_cast<PHINode>(VL0);
3063 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
3064 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3065 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
3067 if (NeedToShuffleReuses) {
3068 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3069 E->ReuseShuffleIndices, "shuffle");
3071 E->VectorizedValue = V;
3073 // PHINodes may have multiple entries from the same block. We want to
3074 // visit every block once.
3075 SmallPtrSet<BasicBlock*, 4> VisitedBBs;
3077 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
3079 BasicBlock *IBB = PH->getIncomingBlock(i);
3081 if (!VisitedBBs.insert(IBB).second) {
3082 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
3086 // Prepare the operand vector.
3087 for (Value *V : E->Scalars)
3088 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
3090 Builder.SetInsertPoint(IBB->getTerminator());
3091 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3092 Value *Vec = vectorizeTree(Operands);
3093 NewPhi->addIncoming(Vec, IBB);
3096 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
3097 "Invalid number of incoming values");
3101 case Instruction::ExtractElement: {
3102 if (!E->NeedToGather) {
3103 Value *V = VL0->getOperand(0);
3104 if (!E->ReorderIndices.empty()) {
3106 inversePermutation(E->ReorderIndices, Mask);
3107 Builder.SetInsertPoint(VL0);
3108 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
3111 if (NeedToShuffleReuses) {
3112 // TODO: Merge this shuffle with the ReorderShuffleMask.
3113 if (E->ReorderIndices.empty())
3114 Builder.SetInsertPoint(VL0);
3115 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3116 E->ReuseShuffleIndices, "shuffle");
3118 E->VectorizedValue = V;
3121 setInsertPointAfterBundle(E->Scalars, S);
3122 auto *V = Gather(E->Scalars, VecTy);
3123 if (NeedToShuffleReuses) {
3124 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3125 E->ReuseShuffleIndices, "shuffle");
3126 if (auto *I = dyn_cast<Instruction>(V)) {
3127 GatherSeq.insert(I);
3128 CSEBlocks.insert(I->getParent());
3131 E->VectorizedValue = V;
3134 case Instruction::ExtractValue: {
3135 if (!E->NeedToGather) {
3136 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3137 Builder.SetInsertPoint(LI);
3138 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3139 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3140 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
3141 Value *NewV = propagateMetadata(V, E->Scalars);
3142 if (!E->ReorderIndices.empty()) {
3144 inversePermutation(E->ReorderIndices, Mask);
3145 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
3148 if (NeedToShuffleReuses) {
3149 // TODO: Merge this shuffle with the ReorderShuffleMask.
3150 NewV = Builder.CreateShuffleVector(
3151 NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3153 E->VectorizedValue = NewV;
3156 setInsertPointAfterBundle(E->Scalars, S);
3157 auto *V = Gather(E->Scalars, VecTy);
3158 if (NeedToShuffleReuses) {
3159 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3160 E->ReuseShuffleIndices, "shuffle");
3161 if (auto *I = dyn_cast<Instruction>(V)) {
3162 GatherSeq.insert(I);
3163 CSEBlocks.insert(I->getParent());
3166 E->VectorizedValue = V;
3169 case Instruction::ZExt:
3170 case Instruction::SExt:
3171 case Instruction::FPToUI:
3172 case Instruction::FPToSI:
3173 case Instruction::FPExt:
3174 case Instruction::PtrToInt:
3175 case Instruction::IntToPtr:
3176 case Instruction::SIToFP:
3177 case Instruction::UIToFP:
3178 case Instruction::Trunc:
3179 case Instruction::FPTrunc:
3180 case Instruction::BitCast: {
3182 for (Value *V : E->Scalars)
3183 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3185 setInsertPointAfterBundle(E->Scalars, S);
3187 Value *InVec = vectorizeTree(INVL);
3189 if (E->VectorizedValue) {
3190 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3191 return E->VectorizedValue;
3194 CastInst *CI = dyn_cast<CastInst>(VL0);
3195 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3196 if (NeedToShuffleReuses) {
3197 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3198 E->ReuseShuffleIndices, "shuffle");
3200 E->VectorizedValue = V;
3201 ++NumVectorInstructions;
3204 case Instruction::FCmp:
3205 case Instruction::ICmp: {
3206 ValueList LHSV, RHSV;
3207 for (Value *V : E->Scalars) {
3208 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3209 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3212 setInsertPointAfterBundle(E->Scalars, S);
3214 Value *L = vectorizeTree(LHSV);
3215 Value *R = vectorizeTree(RHSV);
3217 if (E->VectorizedValue) {
3218 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3219 return E->VectorizedValue;
3222 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3224 if (S.getOpcode() == Instruction::FCmp)
3225 V = Builder.CreateFCmp(P0, L, R);
3227 V = Builder.CreateICmp(P0, L, R);
3229 propagateIRFlags(V, E->Scalars, VL0);
3230 if (NeedToShuffleReuses) {
3231 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3232 E->ReuseShuffleIndices, "shuffle");
3234 E->VectorizedValue = V;
3235 ++NumVectorInstructions;
3238 case Instruction::Select: {
3239 ValueList TrueVec, FalseVec, CondVec;
3240 for (Value *V : E->Scalars) {
3241 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3242 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3243 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3246 setInsertPointAfterBundle(E->Scalars, S);
3248 Value *Cond = vectorizeTree(CondVec);
3249 Value *True = vectorizeTree(TrueVec);
3250 Value *False = vectorizeTree(FalseVec);
3252 if (E->VectorizedValue) {
3253 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3254 return E->VectorizedValue;
3257 Value *V = Builder.CreateSelect(Cond, True, False);
3258 if (NeedToShuffleReuses) {
3259 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3260 E->ReuseShuffleIndices, "shuffle");
3262 E->VectorizedValue = V;
3263 ++NumVectorInstructions;
3266 case Instruction::Add:
3267 case Instruction::FAdd:
3268 case Instruction::Sub:
3269 case Instruction::FSub:
3270 case Instruction::Mul:
3271 case Instruction::FMul:
3272 case Instruction::UDiv:
3273 case Instruction::SDiv:
3274 case Instruction::FDiv:
3275 case Instruction::URem:
3276 case Instruction::SRem:
3277 case Instruction::FRem:
3278 case Instruction::Shl:
3279 case Instruction::LShr:
3280 case Instruction::AShr:
3281 case Instruction::And:
3282 case Instruction::Or:
3283 case Instruction::Xor: {
3284 ValueList LHSVL, RHSVL;
3285 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3286 reorderInputsAccordingToOpcode(S.getOpcode(), E->Scalars, LHSVL,
3289 for (Value *V : E->Scalars) {
3290 auto *I = cast<Instruction>(V);
3291 LHSVL.push_back(I->getOperand(0));
3292 RHSVL.push_back(I->getOperand(1));
3295 setInsertPointAfterBundle(E->Scalars, S);
3297 Value *LHS = vectorizeTree(LHSVL);
3298 Value *RHS = vectorizeTree(RHSVL);
3300 if (E->VectorizedValue) {
3301 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3302 return E->VectorizedValue;
3305 Value *V = Builder.CreateBinOp(
3306 static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3307 propagateIRFlags(V, E->Scalars, VL0);
3308 if (auto *I = dyn_cast<Instruction>(V))
3309 V = propagateMetadata(I, E->Scalars);
3311 if (NeedToShuffleReuses) {
3312 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3313 E->ReuseShuffleIndices, "shuffle");
3315 E->VectorizedValue = V;
3316 ++NumVectorInstructions;
3320 case Instruction::Load: {
3321 // Loads are inserted at the head of the tree because we don't want to
3322 // sink them all the way down past store instructions.
3323 bool IsReorder = !E->ReorderIndices.empty();
3325 S = getSameOpcode(E->Scalars, E->ReorderIndices.front());
3326 VL0 = cast<Instruction>(S.OpValue);
3328 setInsertPointAfterBundle(E->Scalars, S);
3330 LoadInst *LI = cast<LoadInst>(VL0);
3331 Type *ScalarLoadTy = LI->getType();
3332 unsigned AS = LI->getPointerAddressSpace();
3334 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3335 VecTy->getPointerTo(AS));
3337 // The pointer operand uses an in-tree scalar so we add the new BitCast to
3338 // ExternalUses list to make sure that an extract will be generated in the
3340 Value *PO = LI->getPointerOperand();
3341 if (getTreeEntry(PO))
3342 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3344 unsigned Alignment = LI->getAlignment();
3345 LI = Builder.CreateLoad(VecPtr);
3347 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3349 LI->setAlignment(Alignment);
3350 Value *V = propagateMetadata(LI, E->Scalars);
3353 inversePermutation(E->ReorderIndices, Mask);
3354 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3355 Mask, "reorder_shuffle");
3357 if (NeedToShuffleReuses) {
3358 // TODO: Merge this shuffle with the ReorderShuffleMask.
3359 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3360 E->ReuseShuffleIndices, "shuffle");
3362 E->VectorizedValue = V;
3363 ++NumVectorInstructions;
3366 case Instruction::Store: {
3367 StoreInst *SI = cast<StoreInst>(VL0);
3368 unsigned Alignment = SI->getAlignment();
3369 unsigned AS = SI->getPointerAddressSpace();
3371 ValueList ScalarStoreValues;
3372 for (Value *V : E->Scalars)
3373 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3375 setInsertPointAfterBundle(E->Scalars, S);
3377 Value *VecValue = vectorizeTree(ScalarStoreValues);
3378 Value *ScalarPtr = SI->getPointerOperand();
3379 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3380 StoreInst *ST = Builder.CreateStore(VecValue, VecPtr);
3382 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3383 // ExternalUses to make sure that an extract will be generated in the
3385 if (getTreeEntry(ScalarPtr))
3386 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3389 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3391 ST->setAlignment(Alignment);
3392 Value *V = propagateMetadata(ST, E->Scalars);
3393 if (NeedToShuffleReuses) {
3394 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3395 E->ReuseShuffleIndices, "shuffle");
3397 E->VectorizedValue = V;
3398 ++NumVectorInstructions;
3401 case Instruction::GetElementPtr: {
3402 setInsertPointAfterBundle(E->Scalars, S);
3405 for (Value *V : E->Scalars)
3406 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3408 Value *Op0 = vectorizeTree(Op0VL);
3410 std::vector<Value *> OpVecs;
3411 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3414 for (Value *V : E->Scalars)
3415 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3417 Value *OpVec = vectorizeTree(OpVL);
3418 OpVecs.push_back(OpVec);
3421 Value *V = Builder.CreateGEP(
3422 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3423 if (Instruction *I = dyn_cast<Instruction>(V))
3424 V = propagateMetadata(I, E->Scalars);
3426 if (NeedToShuffleReuses) {
3427 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3428 E->ReuseShuffleIndices, "shuffle");
3430 E->VectorizedValue = V;
3431 ++NumVectorInstructions;
3435 case Instruction::Call: {
3436 CallInst *CI = cast<CallInst>(VL0);
3437 setInsertPointAfterBundle(E->Scalars, S);
3439 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3440 Value *ScalarArg = nullptr;
3441 if (CI && (FI = CI->getCalledFunction())) {
3442 IID = FI->getIntrinsicID();
3444 std::vector<Value *> OpVecs;
3445 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3447 // ctlz,cttz and powi are special intrinsics whose second argument is
3448 // a scalar. This argument should not be vectorized.
3449 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3450 CallInst *CEI = cast<CallInst>(VL0);
3451 ScalarArg = CEI->getArgOperand(j);
3452 OpVecs.push_back(CEI->getArgOperand(j));
3455 for (Value *V : E->Scalars) {
3456 CallInst *CEI = cast<CallInst>(V);
3457 OpVL.push_back(CEI->getArgOperand(j));
3460 Value *OpVec = vectorizeTree(OpVL);
3461 LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3462 OpVecs.push_back(OpVec);
3465 Module *M = F->getParent();
3466 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3467 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3468 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3469 SmallVector<OperandBundleDef, 1> OpBundles;
3470 CI->getOperandBundlesAsDefs(OpBundles);
3471 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3473 // The scalar argument uses an in-tree scalar so we add the new vectorized
3474 // call to ExternalUses list to make sure that an extract will be
3475 // generated in the future.
3476 if (ScalarArg && getTreeEntry(ScalarArg))
3477 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3479 propagateIRFlags(V, E->Scalars, VL0);
3480 if (NeedToShuffleReuses) {
3481 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3482 E->ReuseShuffleIndices, "shuffle");
3484 E->VectorizedValue = V;
3485 ++NumVectorInstructions;
3488 case Instruction::ShuffleVector: {
3489 ValueList LHSVL, RHSVL;
3490 assert(S.isAltShuffle() &&
3491 ((Instruction::isBinaryOp(S.getOpcode()) &&
3492 Instruction::isBinaryOp(S.getAltOpcode())) ||
3493 (Instruction::isCast(S.getOpcode()) &&
3494 Instruction::isCast(S.getAltOpcode()))) &&
3495 "Invalid Shuffle Vector Operand");
3498 if (Instruction::isBinaryOp(S.getOpcode())) {
3499 reorderAltShuffleOperands(S, E->Scalars, LHSVL, RHSVL);
3500 setInsertPointAfterBundle(E->Scalars, S);
3501 LHS = vectorizeTree(LHSVL);
3502 RHS = vectorizeTree(RHSVL);
3505 for (Value *V : E->Scalars)
3506 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3507 setInsertPointAfterBundle(E->Scalars, S);
3508 LHS = vectorizeTree(INVL);
3511 if (E->VectorizedValue) {
3512 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3513 return E->VectorizedValue;
3517 if (Instruction::isBinaryOp(S.getOpcode())) {
3518 V0 = Builder.CreateBinOp(
3519 static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3520 V1 = Builder.CreateBinOp(
3521 static_cast<Instruction::BinaryOps>(S.getAltOpcode()), LHS, RHS);
3523 V0 = Builder.CreateCast(
3524 static_cast<Instruction::CastOps>(S.getOpcode()), LHS, VecTy);
3525 V1 = Builder.CreateCast(
3526 static_cast<Instruction::CastOps>(S.getAltOpcode()), LHS, VecTy);
3529 // Create shuffle to take alternate operations from the vector.
3530 // Also, gather up main and alt scalar ops to propagate IR flags to
3531 // each vector operation.
3532 ValueList OpScalars, AltScalars;
3533 unsigned e = E->Scalars.size();
3534 SmallVector<Constant *, 8> Mask(e);
3535 for (unsigned i = 0; i < e; ++i) {
3536 auto *OpInst = cast<Instruction>(E->Scalars[i]);
3537 assert(S.isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode");
3538 if (OpInst->getOpcode() == S.getAltOpcode()) {
3539 Mask[i] = Builder.getInt32(e + i);
3540 AltScalars.push_back(E->Scalars[i]);
3542 Mask[i] = Builder.getInt32(i);
3543 OpScalars.push_back(E->Scalars[i]);
3547 Value *ShuffleMask = ConstantVector::get(Mask);
3548 propagateIRFlags(V0, OpScalars);
3549 propagateIRFlags(V1, AltScalars);
3551 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3552 if (Instruction *I = dyn_cast<Instruction>(V))
3553 V = propagateMetadata(I, E->Scalars);
3554 if (NeedToShuffleReuses) {
3555 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3556 E->ReuseShuffleIndices, "shuffle");
3558 E->VectorizedValue = V;
3559 ++NumVectorInstructions;
3564 llvm_unreachable("unknown inst");
3569 Value *BoUpSLP::vectorizeTree() {
3570 ExtraValueToDebugLocsMap ExternallyUsedValues;
3571 return vectorizeTree(ExternallyUsedValues);
3575 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3576 // All blocks must be scheduled before any instructions are inserted.
3577 for (auto &BSIter : BlocksSchedules) {
3578 scheduleBlock(BSIter.second.get());
3581 Builder.SetInsertPoint(&F->getEntryBlock().front());
3582 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3584 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3585 // vectorized root. InstCombine will then rewrite the entire expression. We
3586 // sign extend the extracted values below.
3587 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3588 if (MinBWs.count(ScalarRoot)) {
3589 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3590 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3591 auto BundleWidth = VectorizableTree[0].Scalars.size();
3592 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3593 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3594 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3595 VectorizableTree[0].VectorizedValue = Trunc;
3598 LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()
3601 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3602 // specified by ScalarType.
3603 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3604 if (!MinBWs.count(ScalarRoot))
3606 if (MinBWs[ScalarRoot].second)
3607 return Builder.CreateSExt(Ex, ScalarType);
3608 return Builder.CreateZExt(Ex, ScalarType);
3611 // Extract all of the elements with the external uses.
3612 for (const auto &ExternalUse : ExternalUses) {
3613 Value *Scalar = ExternalUse.Scalar;
3614 llvm::User *User = ExternalUse.User;
3616 // Skip users that we already RAUW. This happens when one instruction
3617 // has multiple uses of the same value.
3618 if (User && !is_contained(Scalar->users(), User))
3620 TreeEntry *E = getTreeEntry(Scalar);
3621 assert(E && "Invalid scalar");
3622 assert(!E->NeedToGather && "Extracting from a gather list");
3624 Value *Vec = E->VectorizedValue;
3625 assert(Vec && "Can't find vectorizable value");
3627 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3628 // If User == nullptr, the Scalar is used as extra arg. Generate
3629 // ExtractElement instruction and update the record for this scalar in
3630 // ExternallyUsedValues.
3632 assert(ExternallyUsedValues.count(Scalar) &&
3633 "Scalar with nullptr as an external user must be registered in "
3634 "ExternallyUsedValues map");
3635 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3636 Builder.SetInsertPoint(VecI->getParent(),
3637 std::next(VecI->getIterator()));
3639 Builder.SetInsertPoint(&F->getEntryBlock().front());
3641 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3642 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3643 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3644 auto &Locs = ExternallyUsedValues[Scalar];
3645 ExternallyUsedValues.insert({Ex, Locs});
3646 ExternallyUsedValues.erase(Scalar);
3647 // Required to update internally referenced instructions.
3648 Scalar->replaceAllUsesWith(Ex);
3652 // Generate extracts for out-of-tree users.
3653 // Find the insertion point for the extractelement lane.
3654 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3655 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3656 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3657 if (PH->getIncomingValue(i) == Scalar) {
3658 Instruction *IncomingTerminator =
3659 PH->getIncomingBlock(i)->getTerminator();
3660 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3661 Builder.SetInsertPoint(VecI->getParent(),
3662 std::next(VecI->getIterator()));
3664 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3666 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3667 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3668 CSEBlocks.insert(PH->getIncomingBlock(i));
3669 PH->setOperand(i, Ex);
3673 Builder.SetInsertPoint(cast<Instruction>(User));
3674 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3675 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3676 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3677 User->replaceUsesOfWith(Scalar, Ex);
3680 Builder.SetInsertPoint(&F->getEntryBlock().front());
3681 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3682 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3683 CSEBlocks.insert(&F->getEntryBlock());
3684 User->replaceUsesOfWith(Scalar, Ex);
3687 LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3690 // For each vectorized value:
3691 for (TreeEntry &EIdx : VectorizableTree) {
3692 TreeEntry *Entry = &EIdx;
3694 // No need to handle users of gathered values.
3695 if (Entry->NeedToGather)
3698 assert(Entry->VectorizedValue && "Can't find vectorizable value");
3701 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3702 Value *Scalar = Entry->Scalars[Lane];
3704 Type *Ty = Scalar->getType();
3705 if (!Ty->isVoidTy()) {
3707 for (User *U : Scalar->users()) {
3708 LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3710 // It is legal to replace users in the ignorelist by undef.
3711 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3712 "Replacing out-of-tree value with undef");
3715 Value *Undef = UndefValue::get(Ty);
3716 Scalar->replaceAllUsesWith(Undef);
3718 LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3719 eraseInstruction(cast<Instruction>(Scalar));
3723 Builder.ClearInsertionPoint();
3725 return VectorizableTree[0].VectorizedValue;
3728 void BoUpSLP::optimizeGatherSequence() {
3729 LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3730 << " gather sequences instructions.\n");
3731 // LICM InsertElementInst sequences.
3732 for (Instruction *I : GatherSeq) {
3733 if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3736 // Check if this block is inside a loop.
3737 Loop *L = LI->getLoopFor(I->getParent());
3741 // Check if it has a preheader.
3742 BasicBlock *PreHeader = L->getLoopPreheader();
3746 // If the vector or the element that we insert into it are
3747 // instructions that are defined in this basic block then we can't
3748 // hoist this instruction.
3749 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3750 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3751 if (Op0 && L->contains(Op0))
3753 if (Op1 && L->contains(Op1))
3756 // We can hoist this instruction. Move it to the pre-header.
3757 I->moveBefore(PreHeader->getTerminator());
3760 // Make a list of all reachable blocks in our CSE queue.
3761 SmallVector<const DomTreeNode *, 8> CSEWorkList;
3762 CSEWorkList.reserve(CSEBlocks.size());
3763 for (BasicBlock *BB : CSEBlocks)
3764 if (DomTreeNode *N = DT->getNode(BB)) {
3765 assert(DT->isReachableFromEntry(N));
3766 CSEWorkList.push_back(N);
3769 // Sort blocks by domination. This ensures we visit a block after all blocks
3770 // dominating it are visited.
3771 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3772 [this](const DomTreeNode *A, const DomTreeNode *B) {
3773 return DT->properlyDominates(A, B);
3776 // Perform O(N^2) search over the gather sequences and merge identical
3777 // instructions. TODO: We can further optimize this scan if we split the
3778 // instructions into different buckets based on the insert lane.
3779 SmallVector<Instruction *, 16> Visited;
3780 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3781 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
3782 "Worklist not sorted properly!");
3783 BasicBlock *BB = (*I)->getBlock();
3784 // For all instructions in blocks containing gather sequences:
3785 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3786 Instruction *In = &*it++;
3787 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3790 // Check if we can replace this instruction with any of the
3791 // visited instructions.
3792 for (Instruction *v : Visited) {
3793 if (In->isIdenticalTo(v) &&
3794 DT->dominates(v->getParent(), In->getParent())) {
3795 In->replaceAllUsesWith(v);
3796 eraseInstruction(In);
3802 assert(!is_contained(Visited, In));
3803 Visited.push_back(In);
3811 // Groups the instructions to a bundle (which is then a single scheduling entity)
3812 // and schedules instructions until the bundle gets ready.
3813 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3815 const InstructionsState &S) {
3816 if (isa<PHINode>(S.OpValue))
3819 // Initialize the instruction bundle.
3820 Instruction *OldScheduleEnd = ScheduleEnd;
3821 ScheduleData *PrevInBundle = nullptr;
3822 ScheduleData *Bundle = nullptr;
3823 bool ReSchedule = false;
3824 LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n");
3826 // Make sure that the scheduling region contains all
3827 // instructions of the bundle.
3828 for (Value *V : VL) {
3829 if (!extendSchedulingRegion(V, S))
3833 for (Value *V : VL) {
3834 ScheduleData *BundleMember = getScheduleData(V);
3835 assert(BundleMember &&
3836 "no ScheduleData for bundle member (maybe not in same basic block)");
3837 if (BundleMember->IsScheduled) {
3838 // A bundle member was scheduled as single instruction before and now
3839 // needs to be scheduled as part of the bundle. We just get rid of the
3840 // existing schedule.
3841 LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3842 << " was already scheduled\n");
3845 assert(BundleMember->isSchedulingEntity() &&
3846 "bundle member already part of other bundle");
3848 PrevInBundle->NextInBundle = BundleMember;
3850 Bundle = BundleMember;
3852 BundleMember->UnscheduledDepsInBundle = 0;
3853 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3855 // Group the instructions to a bundle.
3856 BundleMember->FirstInBundle = Bundle;
3857 PrevInBundle = BundleMember;
3859 if (ScheduleEnd != OldScheduleEnd) {
3860 // The scheduling region got new instructions at the lower end (or it is a
3861 // new region for the first bundle). This makes it necessary to
3862 // recalculate all dependencies.
3863 // It is seldom that this needs to be done a second time after adding the
3864 // initial bundle to the region.
3865 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3866 doForAllOpcodes(I, [](ScheduleData *SD) {
3867 SD->clearDependencies();
3874 initialFillReadyList(ReadyInsts);
3877 LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3878 << BB->getName() << "\n");
3880 calculateDependencies(Bundle, true, SLP);
3882 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3883 // means that there are no cyclic dependencies and we can schedule it.
3884 // Note that's important that we don't "schedule" the bundle yet (see
3885 // cancelScheduling).
3886 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3888 ScheduleData *pickedSD = ReadyInsts.back();
3889 ReadyInsts.pop_back();
3891 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3892 schedule(pickedSD, ReadyInsts);
3895 if (!Bundle->isReady()) {
3896 cancelScheduling(VL, S.OpValue);
3902 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3904 if (isa<PHINode>(OpValue))
3907 ScheduleData *Bundle = getScheduleData(OpValue);
3908 LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3909 assert(!Bundle->IsScheduled &&
3910 "Can't cancel bundle which is already scheduled");
3911 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3912 "tried to unbundle something which is not a bundle");
3914 // Un-bundle: make single instructions out of the bundle.
3915 ScheduleData *BundleMember = Bundle;
3916 while (BundleMember) {
3917 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3918 BundleMember->FirstInBundle = BundleMember;
3919 ScheduleData *Next = BundleMember->NextInBundle;
3920 BundleMember->NextInBundle = nullptr;
3921 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3922 if (BundleMember->UnscheduledDepsInBundle == 0) {
3923 ReadyInsts.insert(BundleMember);
3925 BundleMember = Next;
3929 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3930 // Allocate a new ScheduleData for the instruction.
3931 if (ChunkPos >= ChunkSize) {
3932 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3935 return &(ScheduleDataChunks.back()[ChunkPos++]);
3938 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3939 const InstructionsState &S) {
3940 if (getScheduleData(V, isOneOf(S, V)))
3942 Instruction *I = dyn_cast<Instruction>(V);
3943 assert(I && "bundle member must be an instruction");
3944 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3945 auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool {
3946 ScheduleData *ISD = getScheduleData(I);
3949 assert(isInSchedulingRegion(ISD) &&
3950 "ScheduleData not in scheduling region");
3951 ScheduleData *SD = allocateScheduleDataChunks();
3953 SD->init(SchedulingRegionID, S.OpValue);
3954 ExtraScheduleDataMap[I][S.OpValue] = SD;
3957 if (CheckSheduleForI(I))
3959 if (!ScheduleStart) {
3960 // It's the first instruction in the new region.
3961 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3963 ScheduleEnd = I->getNextNode();
3964 if (isOneOf(S, I) != I)
3965 CheckSheduleForI(I);
3966 assert(ScheduleEnd && "tried to vectorize a terminator?");
3967 LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3970 // Search up and down at the same time, because we don't know if the new
3971 // instruction is above or below the existing scheduling region.
3972 BasicBlock::reverse_iterator UpIter =
3973 ++ScheduleStart->getIterator().getReverse();
3974 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3975 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3976 BasicBlock::iterator LowerEnd = BB->end();
3978 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3979 LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3983 if (UpIter != UpperEnd) {
3984 if (&*UpIter == I) {
3985 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3987 if (isOneOf(S, I) != I)
3988 CheckSheduleForI(I);
3989 LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I
3995 if (DownIter != LowerEnd) {
3996 if (&*DownIter == I) {
3997 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3999 ScheduleEnd = I->getNextNode();
4000 if (isOneOf(S, I) != I)
4001 CheckSheduleForI(I);
4002 assert(ScheduleEnd && "tried to vectorize a terminator?");
4003 LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I
4009 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
4010 "instruction not found in block");
4015 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
4017 ScheduleData *PrevLoadStore,
4018 ScheduleData *NextLoadStore) {
4019 ScheduleData *CurrentLoadStore = PrevLoadStore;
4020 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
4021 ScheduleData *SD = ScheduleDataMap[I];
4023 SD = allocateScheduleDataChunks();
4024 ScheduleDataMap[I] = SD;
4027 assert(!isInSchedulingRegion(SD) &&
4028 "new ScheduleData already in scheduling region");
4029 SD->init(SchedulingRegionID, I);
4031 if (I->mayReadOrWriteMemory() &&
4032 (!isa<IntrinsicInst>(I) ||
4033 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
4034 // Update the linked list of memory accessing instructions.
4035 if (CurrentLoadStore) {
4036 CurrentLoadStore->NextLoadStore = SD;
4038 FirstLoadStoreInRegion = SD;
4040 CurrentLoadStore = SD;
4043 if (NextLoadStore) {
4044 if (CurrentLoadStore)
4045 CurrentLoadStore->NextLoadStore = NextLoadStore;
4047 LastLoadStoreInRegion = CurrentLoadStore;
4051 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
4052 bool InsertInReadyList,
4054 assert(SD->isSchedulingEntity());
4056 SmallVector<ScheduleData *, 10> WorkList;
4057 WorkList.push_back(SD);
4059 while (!WorkList.empty()) {
4060 ScheduleData *SD = WorkList.back();
4061 WorkList.pop_back();
4063 ScheduleData *BundleMember = SD;
4064 while (BundleMember) {
4065 assert(isInSchedulingRegion(BundleMember));
4066 if (!BundleMember->hasValidDependencies()) {
4068 LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember
4070 BundleMember->Dependencies = 0;
4071 BundleMember->resetUnscheduledDeps();
4073 // Handle def-use chain dependencies.
4074 if (BundleMember->OpValue != BundleMember->Inst) {
4075 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
4076 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4077 BundleMember->Dependencies++;
4078 ScheduleData *DestBundle = UseSD->FirstInBundle;
4079 if (!DestBundle->IsScheduled)
4080 BundleMember->incrementUnscheduledDeps(1);
4081 if (!DestBundle->hasValidDependencies())
4082 WorkList.push_back(DestBundle);
4085 for (User *U : BundleMember->Inst->users()) {
4086 if (isa<Instruction>(U)) {
4087 ScheduleData *UseSD = getScheduleData(U);
4088 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4089 BundleMember->Dependencies++;
4090 ScheduleData *DestBundle = UseSD->FirstInBundle;
4091 if (!DestBundle->IsScheduled)
4092 BundleMember->incrementUnscheduledDeps(1);
4093 if (!DestBundle->hasValidDependencies())
4094 WorkList.push_back(DestBundle);
4097 // I'm not sure if this can ever happen. But we need to be safe.
4098 // This lets the instruction/bundle never be scheduled and
4099 // eventually disable vectorization.
4100 BundleMember->Dependencies++;
4101 BundleMember->incrementUnscheduledDeps(1);
4106 // Handle the memory dependencies.
4107 ScheduleData *DepDest = BundleMember->NextLoadStore;
4109 Instruction *SrcInst = BundleMember->Inst;
4110 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
4111 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
4112 unsigned numAliased = 0;
4113 unsigned DistToSrc = 1;
4116 assert(isInSchedulingRegion(DepDest));
4118 // We have two limits to reduce the complexity:
4119 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
4120 // SLP->isAliased (which is the expensive part in this loop).
4121 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
4122 // the whole loop (even if the loop is fast, it's quadratic).
4123 // It's important for the loop break condition (see below) to
4124 // check this limit even between two read-only instructions.
4125 if (DistToSrc >= MaxMemDepDistance ||
4126 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
4127 (numAliased >= AliasedCheckLimit ||
4128 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
4130 // We increment the counter only if the locations are aliased
4131 // (instead of counting all alias checks). This gives a better
4132 // balance between reduced runtime and accurate dependencies.
4135 DepDest->MemoryDependencies.push_back(BundleMember);
4136 BundleMember->Dependencies++;
4137 ScheduleData *DestBundle = DepDest->FirstInBundle;
4138 if (!DestBundle->IsScheduled) {
4139 BundleMember->incrementUnscheduledDeps(1);
4141 if (!DestBundle->hasValidDependencies()) {
4142 WorkList.push_back(DestBundle);
4145 DepDest = DepDest->NextLoadStore;
4147 // Example, explaining the loop break condition: Let's assume our
4148 // starting instruction is i0 and MaxMemDepDistance = 3.
4151 // i0,i1,i2,i3,i4,i5,i6,i7,i8
4154 // MaxMemDepDistance let us stop alias-checking at i3 and we add
4155 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
4156 // Previously we already added dependencies from i3 to i6,i7,i8
4157 // (because of MaxMemDepDistance). As we added a dependency from
4158 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4159 // and we can abort this loop at i6.
4160 if (DistToSrc >= 2 * MaxMemDepDistance)
4166 BundleMember = BundleMember->NextInBundle;
4168 if (InsertInReadyList && SD->isReady()) {
4169 ReadyInsts.push_back(SD);
4170 LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst
4176 void BoUpSLP::BlockScheduling::resetSchedule() {
4177 assert(ScheduleStart &&
4178 "tried to reset schedule on block which has not been scheduled");
4179 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4180 doForAllOpcodes(I, [&](ScheduleData *SD) {
4181 assert(isInSchedulingRegion(SD) &&
4182 "ScheduleData not in scheduling region");
4183 SD->IsScheduled = false;
4184 SD->resetUnscheduledDeps();
4190 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4191 if (!BS->ScheduleStart)
4194 LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
4196 BS->resetSchedule();
4198 // For the real scheduling we use a more sophisticated ready-list: it is
4199 // sorted by the original instruction location. This lets the final schedule
4200 // be as close as possible to the original instruction order.
4201 struct ScheduleDataCompare {
4202 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4203 return SD2->SchedulingPriority < SD1->SchedulingPriority;
4206 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4208 // Ensure that all dependency data is updated and fill the ready-list with
4209 // initial instructions.
4211 int NumToSchedule = 0;
4212 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4213 I = I->getNextNode()) {
4214 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4215 assert(SD->isPartOfBundle() ==
4216 (getTreeEntry(SD->Inst) != nullptr) &&
4217 "scheduler and vectorizer bundle mismatch");
4218 SD->FirstInBundle->SchedulingPriority = Idx++;
4219 if (SD->isSchedulingEntity()) {
4220 BS->calculateDependencies(SD, false, this);
4225 BS->initialFillReadyList(ReadyInsts);
4227 Instruction *LastScheduledInst = BS->ScheduleEnd;
4229 // Do the "real" scheduling.
4230 while (!ReadyInsts.empty()) {
4231 ScheduleData *picked = *ReadyInsts.begin();
4232 ReadyInsts.erase(ReadyInsts.begin());
4234 // Move the scheduled instruction(s) to their dedicated places, if not
4236 ScheduleData *BundleMember = picked;
4237 while (BundleMember) {
4238 Instruction *pickedInst = BundleMember->Inst;
4239 if (LastScheduledInst->getNextNode() != pickedInst) {
4240 BS->BB->getInstList().remove(pickedInst);
4241 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4244 LastScheduledInst = pickedInst;
4245 BundleMember = BundleMember->NextInBundle;
4248 BS->schedule(picked, ReadyInsts);
4251 assert(NumToSchedule == 0 && "could not schedule all instructions");
4253 // Avoid duplicate scheduling of the block.
4254 BS->ScheduleStart = nullptr;
4257 unsigned BoUpSLP::getVectorElementSize(Value *V) {
4258 // If V is a store, just return the width of the stored value without
4259 // traversing the expression tree. This is the common case.
4260 if (auto *Store = dyn_cast<StoreInst>(V))
4261 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4263 // If V is not a store, we can traverse the expression tree to find loads
4264 // that feed it. The type of the loaded value may indicate a more suitable
4265 // width than V's type. We want to base the vector element size on the width
4266 // of memory operations where possible.
4267 SmallVector<Instruction *, 16> Worklist;
4268 SmallPtrSet<Instruction *, 16> Visited;
4269 if (auto *I = dyn_cast<Instruction>(V))
4270 Worklist.push_back(I);
4272 // Traverse the expression tree in bottom-up order looking for loads. If we
4273 // encounter an instruction we don't yet handle, we give up.
4275 auto FoundUnknownInst = false;
4276 while (!Worklist.empty() && !FoundUnknownInst) {
4277 auto *I = Worklist.pop_back_val();
4280 // We should only be looking at scalar instructions here. If the current
4281 // instruction has a vector type, give up.
4282 auto *Ty = I->getType();
4283 if (isa<VectorType>(Ty))
4284 FoundUnknownInst = true;
4286 // If the current instruction is a load, update MaxWidth to reflect the
4287 // width of the loaded value.
4288 else if (isa<LoadInst>(I))
4289 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4291 // Otherwise, we need to visit the operands of the instruction. We only
4292 // handle the interesting cases from buildTree here. If an operand is an
4293 // instruction we haven't yet visited, we add it to the worklist.
4294 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4295 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4296 for (Use &U : I->operands())
4297 if (auto *J = dyn_cast<Instruction>(U.get()))
4298 if (!Visited.count(J))
4299 Worklist.push_back(J);
4302 // If we don't yet handle the instruction, give up.
4304 FoundUnknownInst = true;
4307 // If we didn't encounter a memory access in the expression tree, or if we
4308 // gave up for some reason, just return the width of V.
4309 if (!MaxWidth || FoundUnknownInst)
4310 return DL->getTypeSizeInBits(V->getType());
4312 // Otherwise, return the maximum width we found.
4316 // Determine if a value V in a vectorizable expression Expr can be demoted to a
4317 // smaller type with a truncation. We collect the values that will be demoted
4318 // in ToDemote and additional roots that require investigating in Roots.
4319 static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
4320 SmallVectorImpl<Value *> &ToDemote,
4321 SmallVectorImpl<Value *> &Roots) {
4322 // We can always demote constants.
4323 if (isa<Constant>(V)) {
4324 ToDemote.push_back(V);
4328 // If the value is not an instruction in the expression with only one use, it
4329 // cannot be demoted.
4330 auto *I = dyn_cast<Instruction>(V);
4331 if (!I || !I->hasOneUse() || !Expr.count(I))
4334 switch (I->getOpcode()) {
4336 // We can always demote truncations and extensions. Since truncations can
4337 // seed additional demotion, we save the truncated value.
4338 case Instruction::Trunc:
4339 Roots.push_back(I->getOperand(0));
4341 case Instruction::ZExt:
4342 case Instruction::SExt:
4345 // We can demote certain binary operations if we can demote both of their
4347 case Instruction::Add:
4348 case Instruction::Sub:
4349 case Instruction::Mul:
4350 case Instruction::And:
4351 case Instruction::Or:
4352 case Instruction::Xor:
4353 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4354 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4358 // We can demote selects if we can demote their true and false values.
4359 case Instruction::Select: {
4360 SelectInst *SI = cast<SelectInst>(I);
4361 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4362 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4367 // We can demote phis if we can demote all their incoming operands. Note that
4368 // we don't need to worry about cycles since we ensure single use above.
4369 case Instruction::PHI: {
4370 PHINode *PN = cast<PHINode>(I);
4371 for (Value *IncValue : PN->incoming_values())
4372 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4377 // Otherwise, conservatively give up.
4382 // Record the value that we can demote.
4383 ToDemote.push_back(V);
4387 void BoUpSLP::computeMinimumValueSizes() {
4388 // If there are no external uses, the expression tree must be rooted by a
4389 // store. We can't demote in-memory values, so there is nothing to do here.
4390 if (ExternalUses.empty())
4393 // We only attempt to truncate integer expressions.
4394 auto &TreeRoot = VectorizableTree[0].Scalars;
4395 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4399 // If the expression is not rooted by a store, these roots should have
4400 // external uses. We will rely on InstCombine to rewrite the expression in
4401 // the narrower type. However, InstCombine only rewrites single-use values.
4402 // This means that if a tree entry other than a root is used externally, it
4403 // must have multiple uses and InstCombine will not rewrite it. The code
4404 // below ensures that only the roots are used externally.
4405 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4406 for (auto &EU : ExternalUses)
4407 if (!Expr.erase(EU.Scalar))
4412 // Collect the scalar values of the vectorizable expression. We will use this
4413 // context to determine which values can be demoted. If we see a truncation,
4414 // we mark it as seeding another demotion.
4415 for (auto &Entry : VectorizableTree)
4416 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4418 // Ensure the roots of the vectorizable tree don't form a cycle. They must
4419 // have a single external user that is not in the vectorizable tree.
4420 for (auto *Root : TreeRoot)
4421 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4424 // Conservatively determine if we can actually truncate the roots of the
4425 // expression. Collect the values that can be demoted in ToDemote and
4426 // additional roots that require investigating in Roots.
4427 SmallVector<Value *, 32> ToDemote;
4428 SmallVector<Value *, 4> Roots;
4429 for (auto *Root : TreeRoot)
4430 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4433 // The maximum bit width required to represent all the values that can be
4434 // demoted without loss of precision. It would be safe to truncate the roots
4435 // of the expression to this width.
4436 auto MaxBitWidth = 8u;
4438 // We first check if all the bits of the roots are demanded. If they're not,
4439 // we can truncate the roots to this narrower type.
4440 for (auto *Root : TreeRoot) {
4441 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4442 MaxBitWidth = std::max<unsigned>(
4443 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4446 // True if the roots can be zero-extended back to their original type, rather
4447 // than sign-extended. We know that if the leading bits are not demanded, we
4448 // can safely zero-extend. So we initialize IsKnownPositive to True.
4449 bool IsKnownPositive = true;
4451 // If all the bits of the roots are demanded, we can try a little harder to
4452 // compute a narrower type. This can happen, for example, if the roots are
4453 // getelementptr indices. InstCombine promotes these indices to the pointer
4454 // width. Thus, all their bits are technically demanded even though the
4455 // address computation might be vectorized in a smaller type.
4457 // We start by looking at each entry that can be demoted. We compute the
4458 // maximum bit width required to store the scalar by using ValueTracking to
4459 // compute the number of high-order bits we can truncate.
4460 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
4461 llvm::all_of(TreeRoot, [](Value *R) {
4462 assert(R->hasOneUse() && "Root should have only one use!");
4463 return isa<GetElementPtrInst>(R->user_back());
4467 // Determine if the sign bit of all the roots is known to be zero. If not,
4468 // IsKnownPositive is set to False.
4469 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4470 KnownBits Known = computeKnownBits(R, *DL);
4471 return Known.isNonNegative();
4474 // Determine the maximum number of bits required to store the scalar
4476 for (auto *Scalar : ToDemote) {
4477 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4478 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4479 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4482 // If we can't prove that the sign bit is zero, we must add one to the
4483 // maximum bit width to account for the unknown sign bit. This preserves
4484 // the existing sign bit so we can safely sign-extend the root back to the
4485 // original type. Otherwise, if we know the sign bit is zero, we will
4486 // zero-extend the root instead.
4488 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4489 // one to the maximum bit width will yield a larger-than-necessary
4490 // type. In general, we need to add an extra bit only if we can't
4491 // prove that the upper bit of the original type is equal to the
4492 // upper bit of the proposed smaller type. If these two bits are the
4493 // same (either zero or one) we know that sign-extending from the
4494 // smaller type will result in the same value. Here, since we can't
4495 // yet prove this, we are just making the proposed smaller type
4496 // larger to ensure correctness.
4497 if (!IsKnownPositive)
4501 // Round MaxBitWidth up to the next power-of-two.
4502 if (!isPowerOf2_64(MaxBitWidth))
4503 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4505 // If the maximum bit width we compute is less than the with of the roots'
4506 // type, we can proceed with the narrowing. Otherwise, do nothing.
4507 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4510 // If we can truncate the root, we must collect additional values that might
4511 // be demoted as a result. That is, those seeded by truncations we will
4513 while (!Roots.empty())
4514 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4516 // Finally, map the values we can demote to the maximum bit with we computed.
4517 for (auto *Scalar : ToDemote)
4518 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4523 /// The SLPVectorizer Pass.
4524 struct SLPVectorizer : public FunctionPass {
4525 SLPVectorizerPass Impl;
4527 /// Pass identification, replacement for typeid
4530 explicit SLPVectorizer() : FunctionPass(ID) {
4531 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4534 bool doInitialization(Module &M) override {
4538 bool runOnFunction(Function &F) override {
4539 if (skipFunction(F))
4542 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4543 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4544 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4545 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4546 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4547 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4548 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4549 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4550 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4551 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4553 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4556 void getAnalysisUsage(AnalysisUsage &AU) const override {
4557 FunctionPass::getAnalysisUsage(AU);
4558 AU.addRequired<AssumptionCacheTracker>();
4559 AU.addRequired<ScalarEvolutionWrapperPass>();
4560 AU.addRequired<AAResultsWrapperPass>();
4561 AU.addRequired<TargetTransformInfoWrapperPass>();
4562 AU.addRequired<LoopInfoWrapperPass>();
4563 AU.addRequired<DominatorTreeWrapperPass>();
4564 AU.addRequired<DemandedBitsWrapperPass>();
4565 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4566 AU.addPreserved<LoopInfoWrapperPass>();
4567 AU.addPreserved<DominatorTreeWrapperPass>();
4568 AU.addPreserved<AAResultsWrapperPass>();
4569 AU.addPreserved<GlobalsAAWrapperPass>();
4570 AU.setPreservesCFG();
4574 } // end anonymous namespace
4576 PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4577 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4578 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4579 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4580 auto *AA = &AM.getResult<AAManager>(F);
4581 auto *LI = &AM.getResult<LoopAnalysis>(F);
4582 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4583 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4584 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4585 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4587 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4589 return PreservedAnalyses::all();
4591 PreservedAnalyses PA;
4592 PA.preserveSet<CFGAnalyses>();
4593 PA.preserve<AAManager>();
4594 PA.preserve<GlobalsAA>();
4598 bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4599 TargetTransformInfo *TTI_,
4600 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4601 LoopInfo *LI_, DominatorTree *DT_,
4602 AssumptionCache *AC_, DemandedBits *DB_,
4603 OptimizationRemarkEmitter *ORE_) {
4612 DL = &F.getParent()->getDataLayout();
4616 bool Changed = false;
4618 // If the target claims to have no vector registers don't attempt
4620 if (!TTI->getNumberOfRegisters(true))
4623 // Don't vectorize when the attribute NoImplicitFloat is used.
4624 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4627 LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4629 // Use the bottom up slp vectorizer to construct chains that start with
4630 // store instructions.
4631 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4633 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4634 // delete instructions.
4636 // Scan the blocks in the function in post order.
4637 for (auto BB : post_order(&F.getEntryBlock())) {
4638 collectSeedInstructions(BB);
4640 // Vectorize trees that end at stores.
4641 if (!Stores.empty()) {
4642 LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4643 << " underlying objects.\n");
4644 Changed |= vectorizeStoreChains(R);
4647 // Vectorize trees that end at reductions.
4648 Changed |= vectorizeChainsInBlock(BB, R);
4650 // Vectorize the index computations of getelementptr instructions. This
4651 // is primarily intended to catch gather-like idioms ending at
4652 // non-consecutive loads.
4653 if (!GEPs.empty()) {
4654 LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4655 << " underlying objects.\n");
4656 Changed |= vectorizeGEPIndices(BB, R);
4661 R.optimizeGatherSequence();
4662 LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4663 LLVM_DEBUG(verifyFunction(F));
4668 /// Check that the Values in the slice in VL array are still existent in
4669 /// the WeakTrackingVH array.
4670 /// Vectorization of part of the VL array may cause later values in the VL array
4671 /// to become invalid. We track when this has happened in the WeakTrackingVH
4673 static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4674 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4675 unsigned SliceSize) {
4676 VL = VL.slice(SliceBegin, SliceSize);
4677 VH = VH.slice(SliceBegin, SliceSize);
4678 return !std::equal(VL.begin(), VL.end(), VH.begin());
4681 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4682 unsigned VecRegSize) {
4683 const unsigned ChainLen = Chain.size();
4684 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4686 const unsigned Sz = R.getVectorElementSize(Chain[0]);
4687 const unsigned VF = VecRegSize / Sz;
4689 if (!isPowerOf2_32(Sz) || VF < 2)
4692 // Keep track of values that were deleted by vectorizing in the loop below.
4693 const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4695 bool Changed = false;
4696 // Look for profitable vectorizable trees at all offsets, starting at zero.
4697 for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4699 // Check that a previous iteration of this loop did not delete the Value.
4700 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4703 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4705 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4707 R.buildTree(Operands);
4708 if (R.isTreeTinyAndNotFullyVectorizable())
4711 R.computeMinimumValueSizes();
4713 int Cost = R.getTreeCost();
4715 LLVM_DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF
4717 if (Cost < -SLPCostThreshold) {
4718 LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4720 using namespace ore;
4722 R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4723 cast<StoreInst>(Chain[i]))
4724 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4725 << " and with tree size "
4726 << NV("TreeSize", R.getTreeSize()));
4730 // Move to the next bundle.
4739 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4741 SetVector<StoreInst *> Heads;
4742 SmallDenseSet<StoreInst *> Tails;
4743 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4745 // We may run into multiple chains that merge into a single chain. We mark the
4746 // stores that we vectorized so that we don't visit the same store twice.
4747 BoUpSLP::ValueSet VectorizedStores;
4748 bool Changed = false;
4750 // Do a quadratic search on all of the given stores in reverse order and find
4751 // all of the pairs of stores that follow each other.
4752 SmallVector<unsigned, 16> IndexQueue;
4753 unsigned E = Stores.size();
4754 IndexQueue.resize(E - 1);
4755 for (unsigned I = E; I > 0; --I) {
4756 unsigned Idx = I - 1;
4757 // If a store has multiple consecutive store candidates, search Stores
4758 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4759 // This is because usually pairing with immediate succeeding or preceding
4760 // candidate create the best chance to find slp vectorization opportunity.
4761 unsigned Offset = 1;
4763 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4764 if (Idx >= Offset) {
4765 IndexQueue[Cnt] = Idx - Offset;
4768 if (Idx + Offset < E) {
4769 IndexQueue[Cnt] = Idx + Offset;
4774 for (auto K : IndexQueue) {
4775 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4776 Tails.insert(Stores[Idx]);
4777 Heads.insert(Stores[K]);
4778 ConsecutiveChain[Stores[K]] = Stores[Idx];
4784 // For stores that start but don't end a link in the chain:
4785 for (auto *SI : llvm::reverse(Heads)) {
4786 if (Tails.count(SI))
4789 // We found a store instr that starts a chain. Now follow the chain and try
4791 BoUpSLP::ValueList Operands;
4793 // Collect the chain into a list.
4794 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4795 Operands.push_back(I);
4796 // Move to the next value in the chain.
4797 I = ConsecutiveChain[I];
4800 // FIXME: Is division-by-2 the correct step? Should we assert that the
4801 // register size is a power-of-2?
4802 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4804 if (vectorizeStoreChain(Operands, R, Size)) {
4805 // Mark the vectorized stores so that we don't vectorize them again.
4806 VectorizedStores.insert(Operands.begin(), Operands.end());
4816 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4817 // Initialize the collections. We will make a single pass over the block.
4821 // Visit the store and getelementptr instructions in BB and organize them in
4822 // Stores and GEPs according to the underlying objects of their pointer
4824 for (Instruction &I : *BB) {
4825 // Ignore store instructions that are volatile or have a pointer operand
4826 // that doesn't point to a scalar type.
4827 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4828 if (!SI->isSimple())
4830 if (!isValidElementType(SI->getValueOperand()->getType()))
4832 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4835 // Ignore getelementptr instructions that have more than one index, a
4836 // constant index, or a pointer operand that doesn't point to a scalar
4838 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4839 auto Idx = GEP->idx_begin()->get();
4840 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4842 if (!isValidElementType(Idx->getType()))
4844 if (GEP->getType()->isVectorTy())
4846 GEPs[GEP->getPointerOperand()].push_back(GEP);
4851 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4854 Value *VL[] = { A, B };
4855 return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4858 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4859 int UserCost, bool AllowReorder) {
4863 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = "
4864 << VL.size() << ".\n");
4866 // Check that all of the parts are scalar instructions of the same type,
4867 // we permit an alternate opcode via InstructionsState.
4868 InstructionsState S = getSameOpcode(VL);
4872 Instruction *I0 = cast<Instruction>(S.OpValue);
4873 unsigned Sz = R.getVectorElementSize(I0);
4874 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4875 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4877 R.getORE()->emit([&]() {
4878 return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
4879 << "Cannot SLP vectorize list: vectorization factor "
4880 << "less than 2 is not supported";
4885 for (Value *V : VL) {
4886 Type *Ty = V->getType();
4887 if (!isValidElementType(Ty)) {
4888 // NOTE: the following will give user internal llvm type name, which may
4890 R.getORE()->emit([&]() {
4891 std::string type_str;
4892 llvm::raw_string_ostream rso(type_str);
4894 return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0)
4895 << "Cannot SLP vectorize list: type "
4896 << rso.str() + " is unsupported by vectorizer";
4902 bool Changed = false;
4903 bool CandidateFound = false;
4904 int MinCost = SLPCostThreshold;
4906 // Keep track of values that were deleted by vectorizing in the loop below.
4907 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4909 unsigned NextInst = 0, MaxInst = VL.size();
4910 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4912 // No actual vectorization should happen, if number of parts is the same as
4913 // provided vectorization factor (i.e. the scalar type is used for vector
4914 // code during codegen).
4915 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4916 if (TTI->getNumberOfParts(VecTy) == VF)
4918 for (unsigned I = NextInst; I < MaxInst; ++I) {
4919 unsigned OpsWidth = 0;
4921 if (I + VF > MaxInst)
4922 OpsWidth = MaxInst - I;
4926 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4929 // Check that a previous iteration of this loop did not delete the Value.
4930 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4933 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4935 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4938 Optional<ArrayRef<unsigned>> Order = R.bestOrder();
4939 // TODO: check if we can allow reordering for more cases.
4940 if (AllowReorder && Order) {
4941 // TODO: reorder tree nodes without tree rebuilding.
4942 // Conceptually, there is nothing actually preventing us from trying to
4943 // reorder a larger list. In fact, we do exactly this when vectorizing
4944 // reductions. However, at this point, we only expect to get here when
4945 // there are exactly two operations.
4946 assert(Ops.size() == 2);
4947 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4948 R.buildTree(ReorderedOps, None);
4950 if (R.isTreeTinyAndNotFullyVectorizable())
4953 R.computeMinimumValueSizes();
4954 int Cost = R.getTreeCost() - UserCost;
4955 CandidateFound = true;
4956 MinCost = std::min(MinCost, Cost);
4958 if (Cost < -SLPCostThreshold) {
4959 LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4960 R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4961 cast<Instruction>(Ops[0]))
4962 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4963 << " and with tree size "
4964 << ore::NV("TreeSize", R.getTreeSize()));
4967 // Move to the next bundle.
4975 if (!Changed && CandidateFound) {
4976 R.getORE()->emit([&]() {
4977 return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0)
4978 << "List vectorization was possible but not beneficial with cost "
4979 << ore::NV("Cost", MinCost) << " >= "
4980 << ore::NV("Treshold", -SLPCostThreshold);
4982 } else if (!Changed) {
4983 R.getORE()->emit([&]() {
4984 return OptimizationRemarkMissed(SV_NAME, "NotPossible", I0)
4985 << "Cannot SLP vectorize list: vectorization was impossible"
4986 << " with available vectorization factors";
4992 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4996 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4999 Value *P = I->getParent();
5001 // Vectorize in current basic block only.
5002 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
5003 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
5004 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
5007 // Try to vectorize V.
5008 if (tryToVectorizePair(Op0, Op1, R))
5011 auto *A = dyn_cast<BinaryOperator>(Op0);
5012 auto *B = dyn_cast<BinaryOperator>(Op1);
5014 if (B && B->hasOneUse()) {
5015 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
5016 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
5017 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
5019 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
5024 if (A && A->hasOneUse()) {
5025 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
5026 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
5027 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
5029 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
5035 /// Generate a shuffle mask to be used in a reduction tree.
5037 /// \param VecLen The length of the vector to be reduced.
5038 /// \param NumEltsToRdx The number of elements that should be reduced in the
5040 /// \param IsPairwise Whether the reduction is a pairwise or splitting
5041 /// reduction. A pairwise reduction will generate a mask of
5042 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
5043 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
5044 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
5045 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
5046 bool IsPairwise, bool IsLeft,
5047 IRBuilder<> &Builder) {
5048 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
5050 SmallVector<Constant *, 32> ShuffleMask(
5051 VecLen, UndefValue::get(Builder.getInt32Ty()));
5054 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
5055 for (unsigned i = 0; i != NumEltsToRdx; ++i)
5056 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
5058 // Move the upper half of the vector to the lower half.
5059 for (unsigned i = 0; i != NumEltsToRdx; ++i)
5060 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
5062 return ConstantVector::get(ShuffleMask);
5067 /// Model horizontal reductions.
5069 /// A horizontal reduction is a tree of reduction operations (currently add and
5070 /// fadd) that has operations that can be put into a vector as its leaf.
5071 /// For example, this tree:
5078 /// This tree has "mul" as its reduced values and "+" as its reduction
5079 /// operations. A reduction might be feeding into a store or a binary operation
5094 class HorizontalReduction {
5095 using ReductionOpsType = SmallVector<Value *, 16>;
5096 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
5097 ReductionOpsListType ReductionOps;
5098 SmallVector<Value *, 32> ReducedVals;
5099 // Use map vector to make stable output.
5100 MapVector<Instruction *, Value *> ExtraArgs;
5102 /// Kind of the reduction data.
5103 enum ReductionKind {
5104 RK_None, /// Not a reduction.
5105 RK_Arithmetic, /// Binary reduction data.
5106 RK_Min, /// Minimum reduction data.
5107 RK_UMin, /// Unsigned minimum reduction data.
5108 RK_Max, /// Maximum reduction data.
5109 RK_UMax, /// Unsigned maximum reduction data.
5112 /// Contains info about operation, like its opcode, left and right operands.
5113 class OperationData {
5114 /// Opcode of the instruction.
5115 unsigned Opcode = 0;
5117 /// Left operand of the reduction operation.
5118 Value *LHS = nullptr;
5120 /// Right operand of the reduction operation.
5121 Value *RHS = nullptr;
5123 /// Kind of the reduction operation.
5124 ReductionKind Kind = RK_None;
5126 /// True if float point min/max reduction has no NaNs.
5129 /// Checks if the reduction operation can be vectorized.
5130 bool isVectorizable() const {
5131 return LHS && RHS &&
5132 // We currently only support add/mul/logical && min/max reductions.
5133 ((Kind == RK_Arithmetic &&
5134 (Opcode == Instruction::Add || Opcode == Instruction::FAdd ||
5135 Opcode == Instruction::Mul || Opcode == Instruction::FMul ||
5136 Opcode == Instruction::And || Opcode == Instruction::Or ||
5137 Opcode == Instruction::Xor)) ||
5138 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5139 (Kind == RK_Min || Kind == RK_Max)) ||
5140 (Opcode == Instruction::ICmp &&
5141 (Kind == RK_UMin || Kind == RK_UMax)));
5144 /// Creates reduction operation with the current opcode.
5145 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5146 assert(isVectorizable() &&
5147 "Expected add|fadd or min/max reduction operation.");
5151 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5154 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5155 : Builder.CreateFCmpOLT(LHS, RHS);
5158 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5159 : Builder.CreateFCmpOGT(LHS, RHS);
5162 assert(Opcode == Instruction::ICmp && "Expected integer types.");
5163 Cmp = Builder.CreateICmpULT(LHS, RHS);
5166 assert(Opcode == Instruction::ICmp && "Expected integer types.");
5167 Cmp = Builder.CreateICmpUGT(LHS, RHS);
5170 llvm_unreachable("Unknown reduction operation.");
5172 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5176 explicit OperationData() = default;
5178 /// Construction for reduced values. They are identified by opcode only and
5179 /// don't have associated LHS/RHS values.
5180 explicit OperationData(Value *V) {
5181 if (auto *I = dyn_cast<Instruction>(V))
5182 Opcode = I->getOpcode();
5185 /// Constructor for reduction operations with opcode and its left and
5187 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5189 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5190 assert(Kind != RK_None && "One of the reduction operations is expected.");
5193 explicit operator bool() const { return Opcode; }
5195 /// Get the index of the first operand.
5196 unsigned getFirstOperandIndex() const {
5197 assert(!!*this && "The opcode is not set.");
5211 /// Total number of operands in the reduction operation.
5212 unsigned getNumberOfOperands() const {
5213 assert(Kind != RK_None && !!*this && LHS && RHS &&
5214 "Expected reduction operation.");
5226 llvm_unreachable("Reduction kind is not set");
5229 /// Checks if the operation has the same parent as \p P.
5230 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5231 assert(Kind != RK_None && !!*this && LHS && RHS &&
5232 "Expected reduction operation.");
5234 return I->getParent() == P;
5237 // Arithmetic reduction operation must be used once only.
5238 return I->getParent() == P;
5243 // SelectInst must be used twice while the condition op must have single
5245 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5246 return I->getParent() == P && Cmp && Cmp->getParent() == P;
5251 llvm_unreachable("Reduction kind is not set");
5253 /// Expected number of uses for reduction operations/reduced values.
5254 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5255 assert(Kind != RK_None && !!*this && LHS && RHS &&
5256 "Expected reduction operation.");
5259 return I->hasOneUse();
5264 return I->hasNUses(2) &&
5266 cast<SelectInst>(I)->getCondition()->hasOneUse());
5270 llvm_unreachable("Reduction kind is not set");
5273 /// Initializes the list of reduction operations.
5274 void initReductionOps(ReductionOpsListType &ReductionOps) {
5275 assert(Kind != RK_None && !!*this && LHS && RHS &&
5276 "Expected reduction operation.");
5279 ReductionOps.assign(1, ReductionOpsType());
5285 ReductionOps.assign(2, ReductionOpsType());
5288 llvm_unreachable("Reduction kind is not set");
5291 /// Add all reduction operations for the reduction instruction \p I.
5292 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5293 assert(Kind != RK_None && !!*this && LHS && RHS &&
5294 "Expected reduction operation.");
5297 ReductionOps[0].emplace_back(I);
5303 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5304 ReductionOps[1].emplace_back(I);
5307 llvm_unreachable("Reduction kind is not set");
5311 /// Checks if instruction is associative and can be vectorized.
5312 bool isAssociative(Instruction *I) const {
5313 assert(Kind != RK_None && *this && LHS && RHS &&
5314 "Expected reduction operation.");
5317 return I->isAssociative();
5320 return Opcode == Instruction::ICmp ||
5321 cast<Instruction>(I->getOperand(0))->isFast();
5324 assert(Opcode == Instruction::ICmp &&
5325 "Only integer compare operation is expected.");
5330 llvm_unreachable("Reduction kind is not set");
5333 /// Checks if the reduction operation can be vectorized.
5334 bool isVectorizable(Instruction *I) const {
5335 return isVectorizable() && isAssociative(I);
5338 /// Checks if two operation data are both a reduction op or both a reduced
5340 bool operator==(const OperationData &OD) {
5341 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
5342 "One of the comparing operations is incorrect.");
5343 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5345 bool operator!=(const OperationData &OD) { return !(*this == OD); }
5354 /// Get the opcode of the reduction operation.
5355 unsigned getOpcode() const {
5356 assert(isVectorizable() && "Expected vectorizable operation.");
5360 /// Get kind of reduction data.
5361 ReductionKind getKind() const { return Kind; }
5362 Value *getLHS() const { return LHS; }
5363 Value *getRHS() const { return RHS; }
5364 Type *getConditionType() const {
5372 return CmpInst::makeCmpResultType(LHS->getType());
5376 llvm_unreachable("Reduction kind is not set");
5379 /// Creates reduction operation with the current opcode with the IR flags
5380 /// from \p ReductionOps.
5381 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5382 const ReductionOpsListType &ReductionOps) const {
5383 assert(isVectorizable() &&
5384 "Expected add|fadd or min/max reduction operation.");
5385 auto *Op = createOp(Builder, Name);
5388 propagateIRFlags(Op, ReductionOps[0]);
5394 if (auto *SI = dyn_cast<SelectInst>(Op))
5395 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5396 propagateIRFlags(Op, ReductionOps[1]);
5401 llvm_unreachable("Unknown reduction operation.");
5403 /// Creates reduction operation with the current opcode with the IR flags
5405 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5406 Instruction *I) const {
5407 assert(isVectorizable() &&
5408 "Expected add|fadd or min/max reduction operation.");
5409 auto *Op = createOp(Builder, Name);
5412 propagateIRFlags(Op, I);
5418 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5419 propagateIRFlags(SI->getCondition(),
5420 cast<SelectInst>(I)->getCondition());
5422 propagateIRFlags(Op, I);
5427 llvm_unreachable("Unknown reduction operation.");
5430 TargetTransformInfo::ReductionFlags getFlags() const {
5431 TargetTransformInfo::ReductionFlags Flags;
5432 Flags.NoNaN = NoNaN;
5437 Flags.IsSigned = Opcode == Instruction::ICmp;
5438 Flags.IsMaxOp = false;
5441 Flags.IsSigned = Opcode == Instruction::ICmp;
5442 Flags.IsMaxOp = true;
5445 Flags.IsSigned = false;
5446 Flags.IsMaxOp = false;
5449 Flags.IsSigned = false;
5450 Flags.IsMaxOp = true;
5453 llvm_unreachable("Reduction kind is not set");
5459 WeakTrackingVH ReductionRoot;
5461 /// The operation data of the reduction operation.
5462 OperationData ReductionData;
5464 /// The operation data of the values we perform a reduction on.
5465 OperationData ReducedValueData;
5467 /// Should we model this reduction as a pairwise reduction tree or a tree that
5468 /// splits the vector in halves and adds those halves.
5469 bool IsPairwiseReduction = false;
5471 /// Checks if the ParentStackElem.first should be marked as a reduction
5472 /// operation with an extra argument or as extra argument itself.
5473 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5475 if (ExtraArgs.count(ParentStackElem.first)) {
5476 ExtraArgs[ParentStackElem.first] = nullptr;
5477 // We ran into something like:
5478 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5479 // The whole ParentStackElem.first should be considered as an extra value
5481 // Do not perform analysis of remaining operands of ParentStackElem.first
5482 // instruction, this whole instruction is an extra argument.
5483 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5485 // We ran into something like:
5486 // ParentStackElem.first += ... + ExtraArg + ...
5487 ExtraArgs[ParentStackElem.first] = ExtraArg;
5491 static OperationData getOperationData(Value *V) {
5493 return OperationData();
5497 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5498 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5501 if (auto *Select = dyn_cast<SelectInst>(V)) {
5502 // Look for a min/max pattern.
5503 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5504 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5505 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5506 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5507 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5508 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5509 return OperationData(
5510 Instruction::FCmp, LHS, RHS, RK_Min,
5511 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5512 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5513 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5514 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5515 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5516 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5517 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5518 return OperationData(
5519 Instruction::FCmp, LHS, RHS, RK_Max,
5520 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5522 // Try harder: look for min/max pattern based on instructions producing
5523 // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2).
5524 // During the intermediate stages of SLP, it's very common to have
5525 // pattern like this (since optimizeGatherSequence is run only once
5527 // %1 = extractelement <2 x i32> %a, i32 0
5528 // %2 = extractelement <2 x i32> %a, i32 1
5529 // %cond = icmp sgt i32 %1, %2
5530 // %3 = extractelement <2 x i32> %a, i32 0
5531 // %4 = extractelement <2 x i32> %a, i32 1
5532 // %select = select i1 %cond, i32 %3, i32 %4
5533 CmpInst::Predicate Pred;
5537 LHS = Select->getTrueValue();
5538 RHS = Select->getFalseValue();
5539 Value *Cond = Select->getCondition();
5541 // TODO: Support inverse predicates.
5542 if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) {
5543 if (!isa<ExtractElementInst>(RHS) ||
5544 !L2->isIdenticalTo(cast<Instruction>(RHS)))
5545 return OperationData(V);
5546 } else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) {
5547 if (!isa<ExtractElementInst>(LHS) ||
5548 !L1->isIdenticalTo(cast<Instruction>(LHS)))
5549 return OperationData(V);
5551 if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS))
5552 return OperationData(V);
5553 if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) ||
5554 !L1->isIdenticalTo(cast<Instruction>(LHS)) ||
5555 !L2->isIdenticalTo(cast<Instruction>(RHS)))
5556 return OperationData(V);
5560 return OperationData(V);
5562 case CmpInst::ICMP_ULT:
5563 case CmpInst::ICMP_ULE:
5564 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5566 case CmpInst::ICMP_SLT:
5567 case CmpInst::ICMP_SLE:
5568 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5570 case CmpInst::FCMP_OLT:
5571 case CmpInst::FCMP_OLE:
5572 case CmpInst::FCMP_ULT:
5573 case CmpInst::FCMP_ULE:
5574 return OperationData(Instruction::FCmp, LHS, RHS, RK_Min,
5575 cast<Instruction>(Cond)->hasNoNaNs());
5577 case CmpInst::ICMP_UGT:
5578 case CmpInst::ICMP_UGE:
5579 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5581 case CmpInst::ICMP_SGT:
5582 case CmpInst::ICMP_SGE:
5583 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5585 case CmpInst::FCMP_OGT:
5586 case CmpInst::FCMP_OGE:
5587 case CmpInst::FCMP_UGT:
5588 case CmpInst::FCMP_UGE:
5589 return OperationData(Instruction::FCmp, LHS, RHS, RK_Max,
5590 cast<Instruction>(Cond)->hasNoNaNs());
5594 return OperationData(V);
5598 HorizontalReduction() = default;
5600 /// Try to find a reduction tree.
5601 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5602 assert((!Phi || is_contained(Phi->operands(), B)) &&
5603 "Thi phi needs to use the binary operator");
5605 ReductionData = getOperationData(B);
5607 // We could have a initial reductions that is not an add.
5608 // r *= v1 + v2 + v3 + v4
5609 // In such a case start looking for a tree rooted in the first '+'.
5611 if (ReductionData.getLHS() == Phi) {
5613 B = dyn_cast<Instruction>(ReductionData.getRHS());
5614 ReductionData = getOperationData(B);
5615 } else if (ReductionData.getRHS() == Phi) {
5617 B = dyn_cast<Instruction>(ReductionData.getLHS());
5618 ReductionData = getOperationData(B);
5622 if (!ReductionData.isVectorizable(B))
5625 Type *Ty = B->getType();
5626 if (!isValidElementType(Ty))
5628 if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy())
5631 ReducedValueData.clear();
5634 // Post order traverse the reduction tree starting at B. We only handle true
5635 // trees containing only binary operators.
5636 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5637 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5638 ReductionData.initReductionOps(ReductionOps);
5639 while (!Stack.empty()) {
5640 Instruction *TreeN = Stack.back().first;
5641 unsigned EdgeToVist = Stack.back().second++;
5642 OperationData OpData = getOperationData(TreeN);
5643 bool IsReducedValue = OpData != ReductionData;
5646 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5648 ReducedVals.push_back(TreeN);
5650 auto I = ExtraArgs.find(TreeN);
5651 if (I != ExtraArgs.end() && !I->second) {
5652 // Check if TreeN is an extra argument of its parent operation.
5653 if (Stack.size() <= 1) {
5654 // TreeN can't be an extra argument as it is a root reduction
5658 // Yes, TreeN is an extra argument, do not add it to a list of
5659 // reduction operations.
5660 // Stack[Stack.size() - 2] always points to the parent operation.
5661 markExtraArg(Stack[Stack.size() - 2], TreeN);
5662 ExtraArgs.erase(TreeN);
5664 ReductionData.addReductionOps(TreeN, ReductionOps);
5671 // Visit left or right.
5672 Value *NextV = TreeN->getOperand(EdgeToVist);
5674 auto *I = dyn_cast<Instruction>(NextV);
5675 OpData = getOperationData(I);
5676 // Continue analysis if the next operand is a reduction operation or
5677 // (possibly) a reduced value. If the reduced value opcode is not set,
5678 // the first met operation != reduction operation is considered as the
5679 // reduced value class.
5680 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5681 OpData == ReductionData)) {
5682 const bool IsReductionOperation = OpData == ReductionData;
5683 // Only handle trees in the current basic block.
5684 if (!ReductionData.hasSameParent(I, B->getParent(),
5685 IsReductionOperation)) {
5686 // I is an extra argument for TreeN (its parent operation).
5687 markExtraArg(Stack.back(), I);
5691 // Each tree node needs to have minimal number of users except for the
5692 // ultimate reduction.
5693 if (!ReductionData.hasRequiredNumberOfUses(I,
5694 OpData == ReductionData) &&
5696 // I is an extra argument for TreeN (its parent operation).
5697 markExtraArg(Stack.back(), I);
5701 if (IsReductionOperation) {
5702 // We need to be able to reassociate the reduction operations.
5703 if (!OpData.isAssociative(I)) {
5704 // I is an extra argument for TreeN (its parent operation).
5705 markExtraArg(Stack.back(), I);
5708 } else if (ReducedValueData &&
5709 ReducedValueData != OpData) {
5710 // Make sure that the opcodes of the operations that we are going to
5712 // I is an extra argument for TreeN (its parent operation).
5713 markExtraArg(Stack.back(), I);
5715 } else if (!ReducedValueData)
5716 ReducedValueData = OpData;
5718 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5722 // NextV is an extra argument for TreeN (its parent operation).
5723 markExtraArg(Stack.back(), NextV);
5728 /// Attempt to vectorize the tree found by
5729 /// matchAssociativeReduction.
5730 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5731 if (ReducedVals.empty())
5734 // If there is a sufficient number of reduction values, reduce
5735 // to a nearby power-of-2. Can safely generate oversized
5736 // vectors and rely on the backend to split them to legal sizes.
5737 unsigned NumReducedVals = ReducedVals.size();
5738 if (NumReducedVals < 4)
5741 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5743 Value *VectorizedTree = nullptr;
5744 IRBuilder<> Builder(cast<Instruction>(ReductionRoot));
5745 FastMathFlags Unsafe;
5747 Builder.setFastMathFlags(Unsafe);
5750 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5751 // The same extra argument may be used several time, so log each attempt
5753 for (auto &Pair : ExtraArgs) {
5754 assert(Pair.first && "DebugLoc must be set.");
5755 ExternallyUsedValues[Pair.second].push_back(Pair.first);
5757 // The reduction root is used as the insertion point for new instructions,
5758 // so set it as externally used to prevent it from being deleted.
5759 ExternallyUsedValues[ReductionRoot];
5760 SmallVector<Value *, 16> IgnoreList;
5761 for (auto &V : ReductionOps)
5762 IgnoreList.append(V.begin(), V.end());
5763 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5764 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5765 V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5766 Optional<ArrayRef<unsigned>> Order = V.bestOrder();
5767 // TODO: Handle orders of size less than number of elements in the vector.
5768 if (Order && Order->size() == VL.size()) {
5769 // TODO: reorder tree nodes without tree rebuilding.
5770 SmallVector<Value *, 4> ReorderedOps(VL.size());
5771 llvm::transform(*Order, ReorderedOps.begin(),
5772 [VL](const unsigned Idx) { return VL[Idx]; });
5773 V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList);
5775 if (V.isTreeTinyAndNotFullyVectorizable())
5778 V.computeMinimumValueSizes();
5781 int TreeCost = V.getTreeCost();
5782 int ReductionCost = getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5783 int Cost = TreeCost + ReductionCost;
5784 if (Cost >= -SLPCostThreshold) {
5785 V.getORE()->emit([&]() {
5786 return OptimizationRemarkMissed(
5787 SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5788 << "Vectorizing horizontal reduction is possible"
5789 << "but not beneficial with cost "
5790 << ore::NV("Cost", Cost) << " and threshold "
5791 << ore::NV("Threshold", -SLPCostThreshold);
5796 LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
5797 << Cost << ". (HorRdx)\n");
5798 V.getORE()->emit([&]() {
5799 return OptimizationRemark(
5800 SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5801 << "Vectorized horizontal reduction with cost "
5802 << ore::NV("Cost", Cost) << " and with tree size "
5803 << ore::NV("TreeSize", V.getTreeSize());
5806 // Vectorize a tree.
5807 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5808 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5810 // Emit a reduction.
5811 Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
5812 Value *ReducedSubTree =
5813 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5814 if (VectorizedTree) {
5815 Builder.SetCurrentDebugLocation(Loc);
5816 OperationData VectReductionData(ReductionData.getOpcode(),
5817 VectorizedTree, ReducedSubTree,
5818 ReductionData.getKind());
5820 VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5822 VectorizedTree = ReducedSubTree;
5824 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5827 if (VectorizedTree) {
5828 // Finish the reduction.
5829 for (; i < NumReducedVals; ++i) {
5830 auto *I = cast<Instruction>(ReducedVals[i]);
5831 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5832 OperationData VectReductionData(ReductionData.getOpcode(),
5834 ReductionData.getKind());
5835 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5837 for (auto &Pair : ExternallyUsedValues) {
5838 // Add each externally used value to the final reduction.
5839 for (auto *I : Pair.second) {
5840 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5841 OperationData VectReductionData(ReductionData.getOpcode(),
5842 VectorizedTree, Pair.first,
5843 ReductionData.getKind());
5844 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5848 ReductionRoot->replaceAllUsesWith(VectorizedTree);
5850 return VectorizedTree != nullptr;
5853 unsigned numReductionValues() const {
5854 return ReducedVals.size();
5858 /// Calculate the cost of a reduction.
5859 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5860 unsigned ReduxWidth) {
5861 Type *ScalarTy = FirstReducedVal->getType();
5862 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5864 int PairwiseRdxCost;
5865 int SplittingRdxCost;
5866 switch (ReductionData.getKind()) {
5869 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5870 /*IsPairwiseForm=*/true);
5872 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5873 /*IsPairwiseForm=*/false);
5879 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5880 bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5881 ReductionData.getKind() == RK_UMax;
5883 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5884 /*IsPairwiseForm=*/true, IsUnsigned);
5886 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5887 /*IsPairwiseForm=*/false, IsUnsigned);
5891 llvm_unreachable("Expected arithmetic or min/max reduction operation");
5894 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5895 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5897 int ScalarReduxCost;
5898 switch (ReductionData.getKind()) {
5901 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5908 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5909 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5910 CmpInst::makeCmpResultType(ScalarTy));
5913 llvm_unreachable("Expected arithmetic or min/max reduction operation");
5915 ScalarReduxCost *= (ReduxWidth - 1);
5917 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost
5918 << " for reduction that starts with " << *FirstReducedVal
5920 << (IsPairwiseReduction ? "pairwise" : "splitting")
5921 << " reduction)\n");
5923 return VecReduxCost - ScalarReduxCost;
5926 /// Emit a horizontal reduction of the vectorized value.
5927 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5928 unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5929 assert(VectorizedValue && "Need to have a vectorized tree node");
5930 assert(isPowerOf2_32(ReduxWidth) &&
5931 "We only handle power-of-two reductions for now");
5933 if (!IsPairwiseReduction)
5934 return createSimpleTargetReduction(
5935 Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5936 ReductionData.getFlags(), ReductionOps.back());
5938 Value *TmpVec = VectorizedValue;
5939 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5941 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5943 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5945 Value *LeftShuf = Builder.CreateShuffleVector(
5946 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5947 Value *RightShuf = Builder.CreateShuffleVector(
5948 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5950 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5951 RightShuf, ReductionData.getKind());
5952 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5955 // The result is in the first element of the vector.
5956 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5960 } // end anonymous namespace
5962 /// Recognize construction of vectors like
5963 /// %ra = insertelement <4 x float> undef, float %s0, i32 0
5964 /// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5965 /// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5966 /// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5967 /// starting from the last insertelement instruction.
5969 /// Returns true if it matches
5970 static bool findBuildVector(InsertElementInst *LastInsertElem,
5971 TargetTransformInfo *TTI,
5972 SmallVectorImpl<Value *> &BuildVectorOpds,
5977 if (auto *CI = dyn_cast<ConstantInt>(LastInsertElem->getOperand(2))) {
5978 UserCost += TTI->getVectorInstrCost(Instruction::InsertElement,
5979 LastInsertElem->getType(),
5980 CI->getZExtValue());
5982 BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5983 V = LastInsertElem->getOperand(0);
5984 if (isa<UndefValue>(V))
5986 LastInsertElem = dyn_cast<InsertElementInst>(V);
5987 if (!LastInsertElem || !LastInsertElem->hasOneUse())
5990 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5994 /// Like findBuildVector, but looks for construction of aggregate.
5996 /// \return true if it matches.
5997 static bool findBuildAggregate(InsertValueInst *IV,
5998 SmallVectorImpl<Value *> &BuildVectorOpds) {
6001 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
6002 V = IV->getAggregateOperand();
6003 if (isa<UndefValue>(V))
6005 IV = dyn_cast<InsertValueInst>(V);
6006 if (!IV || !IV->hasOneUse())
6009 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
6013 static bool PhiTypeSorterFunc(Value *V, Value *V2) {
6014 return V->getType() < V2->getType();
6017 /// Try and get a reduction value from a phi node.
6019 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions
6020 /// if they come from either \p ParentBB or a containing loop latch.
6022 /// \returns A candidate reduction value if possible, or \code nullptr \endcode
6023 /// if not possible.
6024 static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
6025 BasicBlock *ParentBB, LoopInfo *LI) {
6026 // There are situations where the reduction value is not dominated by the
6027 // reduction phi. Vectorizing such cases has been reported to cause
6028 // miscompiles. See PR25787.
6029 auto DominatedReduxValue = [&](Value *R) {
6030 return isa<Instruction>(R) &&
6031 DT->dominates(P->getParent(), cast<Instruction>(R)->getParent());
6034 Value *Rdx = nullptr;
6036 // Return the incoming value if it comes from the same BB as the phi node.
6037 if (P->getIncomingBlock(0) == ParentBB) {
6038 Rdx = P->getIncomingValue(0);
6039 } else if (P->getIncomingBlock(1) == ParentBB) {
6040 Rdx = P->getIncomingValue(1);
6043 if (Rdx && DominatedReduxValue(Rdx))
6046 // Otherwise, check whether we have a loop latch to look at.
6047 Loop *BBL = LI->getLoopFor(ParentBB);
6050 BasicBlock *BBLatch = BBL->getLoopLatch();
6054 // There is a loop latch, return the incoming value if it comes from
6055 // that. This reduction pattern occasionally turns up.
6056 if (P->getIncomingBlock(0) == BBLatch) {
6057 Rdx = P->getIncomingValue(0);
6058 } else if (P->getIncomingBlock(1) == BBLatch) {
6059 Rdx = P->getIncomingValue(1);
6062 if (Rdx && DominatedReduxValue(Rdx))
6068 /// Attempt to reduce a horizontal reduction.
6069 /// If it is legal to match a horizontal reduction feeding the phi node \a P
6070 /// with reduction operators \a Root (or one of its operands) in a basic block
6071 /// \a BB, then check if it can be done. If horizontal reduction is not found
6072 /// and root instruction is a binary operation, vectorization of the operands is
6074 /// \returns true if a horizontal reduction was matched and reduced or operands
6075 /// of one of the binary instruction were vectorized.
6076 /// \returns false if a horizontal reduction was not matched (or not possible)
6077 /// or no vectorization of any binary operation feeding \a Root instruction was
6079 static bool tryToVectorizeHorReductionOrInstOperands(
6080 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
6081 TargetTransformInfo *TTI,
6082 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
6083 if (!ShouldVectorizeHor)
6089 if (Root->getParent() != BB || isa<PHINode>(Root))
6091 // Start analysis starting from Root instruction. If horizontal reduction is
6092 // found, try to vectorize it. If it is not a horizontal reduction or
6093 // vectorization is not possible or not effective, and currently analyzed
6094 // instruction is a binary operation, try to vectorize the operands, using
6095 // pre-order DFS traversal order. If the operands were not vectorized, repeat
6096 // the same procedure considering each operand as a possible root of the
6097 // horizontal reduction.
6098 // Interrupt the process if the Root instruction itself was vectorized or all
6099 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
6100 SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
6101 SmallPtrSet<Value *, 8> VisitedInstrs;
6103 while (!Stack.empty()) {
6106 std::tie(V, Level) = Stack.pop_back_val();
6109 auto *Inst = dyn_cast<Instruction>(V);
6112 auto *BI = dyn_cast<BinaryOperator>(Inst);
6113 auto *SI = dyn_cast<SelectInst>(Inst);
6115 HorizontalReduction HorRdx;
6116 if (HorRdx.matchAssociativeReduction(P, Inst)) {
6117 if (HorRdx.tryToReduce(R, TTI)) {
6119 // Set P to nullptr to avoid re-analysis of phi node in
6120 // matchAssociativeReduction function unless this is the root node.
6126 Inst = dyn_cast<Instruction>(BI->getOperand(0));
6128 Inst = dyn_cast<Instruction>(BI->getOperand(1));
6130 // Set P to nullptr to avoid re-analysis of phi node in
6131 // matchAssociativeReduction function unless this is the root node.
6137 // Set P to nullptr to avoid re-analysis of phi node in
6138 // matchAssociativeReduction function unless this is the root node.
6140 if (Vectorize(Inst, R)) {
6145 // Try to vectorize operands.
6146 // Continue analysis for the instruction from the same basic block only to
6147 // save compile time.
6148 if (++Level < RecursionMaxDepth)
6149 for (auto *Op : Inst->operand_values())
6150 if (VisitedInstrs.insert(Op).second)
6151 if (auto *I = dyn_cast<Instruction>(Op))
6152 if (!isa<PHINode>(I) && I->getParent() == BB)
6153 Stack.emplace_back(Op, Level);
6158 bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
6159 BasicBlock *BB, BoUpSLP &R,
6160 TargetTransformInfo *TTI) {
6163 auto *I = dyn_cast<Instruction>(V);
6167 if (!isa<BinaryOperator>(I))
6169 // Try to match and vectorize a horizontal reduction.
6170 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
6171 return tryToVectorize(I, R);
6173 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
6174 ExtraVectorization);
6177 bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
6178 BasicBlock *BB, BoUpSLP &R) {
6179 const DataLayout &DL = BB->getModule()->getDataLayout();
6180 if (!R.canMapToVector(IVI->getType(), DL))
6183 SmallVector<Value *, 16> BuildVectorOpds;
6184 if (!findBuildAggregate(IVI, BuildVectorOpds))
6187 LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
6188 // Aggregate value is unlikely to be processed in vector register, we need to
6189 // extract scalars into scalar registers, so NeedExtraction is set true.
6190 return tryToVectorizeList(BuildVectorOpds, R);
6193 bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
6194 BasicBlock *BB, BoUpSLP &R) {
6196 SmallVector<Value *, 16> BuildVectorOpds;
6197 if (!findBuildVector(IEI, TTI, BuildVectorOpds, UserCost) ||
6198 (llvm::all_of(BuildVectorOpds,
6199 [](Value *V) { return isa<ExtractElementInst>(V); }) &&
6200 isShuffle(BuildVectorOpds)))
6203 // Vectorize starting with the build vector operands ignoring the BuildVector
6204 // instructions for the purpose of scheduling and user extraction.
6205 return tryToVectorizeList(BuildVectorOpds, R, UserCost);
6208 bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
6210 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
6213 bool OpsChanged = false;
6214 for (int Idx = 0; Idx < 2; ++Idx) {
6216 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
6221 bool SLPVectorizerPass::vectorizeSimpleInstructions(
6222 SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
6223 bool OpsChanged = false;
6224 for (auto &VH : reverse(Instructions)) {
6225 auto *I = dyn_cast_or_null<Instruction>(VH);
6228 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
6229 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
6230 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
6231 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
6232 else if (auto *CI = dyn_cast<CmpInst>(I))
6233 OpsChanged |= vectorizeCmpInst(CI, BB, R);
6235 Instructions.clear();
6239 bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
6240 bool Changed = false;
6241 SmallVector<Value *, 4> Incoming;
6242 SmallPtrSet<Value *, 16> VisitedInstrs;
6244 bool HaveVectorizedPhiNodes = true;
6245 while (HaveVectorizedPhiNodes) {
6246 HaveVectorizedPhiNodes = false;
6248 // Collect the incoming values from the PHIs.
6250 for (Instruction &I : *BB) {
6251 PHINode *P = dyn_cast<PHINode>(&I);
6255 if (!VisitedInstrs.count(P))
6256 Incoming.push_back(P);
6260 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
6262 // Try to vectorize elements base on their type.
6263 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
6267 // Look for the next elements with the same type.
6268 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
6269 while (SameTypeIt != E &&
6270 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
6271 VisitedInstrs.insert(*SameTypeIt);
6275 // Try to vectorize them.
6276 unsigned NumElts = (SameTypeIt - IncIt);
6277 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize starting at PHIs ("
6278 << NumElts << ")\n");
6279 // The order in which the phi nodes appear in the program does not matter.
6280 // So allow tryToVectorizeList to reorder them if it is beneficial. This
6281 // is done when there are exactly two elements since tryToVectorizeList
6282 // asserts that there are only two values when AllowReorder is true.
6283 bool AllowReorder = NumElts == 2;
6284 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
6285 /*UserCost=*/0, AllowReorder)) {
6286 // Success start over because instructions might have been changed.
6287 HaveVectorizedPhiNodes = true;
6292 // Start over at the next instruction of a different type (or the end).
6297 VisitedInstrs.clear();
6299 SmallVector<WeakVH, 8> PostProcessInstructions;
6300 SmallDenseSet<Instruction *, 4> KeyNodes;
6301 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
6302 // We may go through BB multiple times so skip the one we have checked.
6303 if (!VisitedInstrs.insert(&*it).second) {
6304 if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
6305 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
6306 // We would like to start over since some instructions are deleted
6307 // and the iterator may become invalid value.
6315 if (isa<DbgInfoIntrinsic>(it))
6318 // Try to vectorize reductions that use PHINodes.
6319 if (PHINode *P = dyn_cast<PHINode>(it)) {
6320 // Check that the PHI is a reduction PHI.
6321 if (P->getNumIncomingValues() != 2)
6324 // Try to match and vectorize a horizontal reduction.
6325 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
6335 // Ran into an instruction without users, like terminator, or function call
6336 // with ignored return value, store. Ignore unused instructions (basing on
6337 // instruction type, except for CallInst and InvokeInst).
6338 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
6339 isa<InvokeInst>(it))) {
6340 KeyNodes.insert(&*it);
6341 bool OpsChanged = false;
6342 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
6343 for (auto *V : it->operand_values()) {
6344 // Try to match and vectorize a horizontal reduction.
6345 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
6348 // Start vectorization of post-process list of instructions from the
6349 // top-tree instructions to try to vectorize as many instructions as
6351 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
6353 // We would like to start over since some instructions are deleted
6354 // and the iterator may become invalid value.
6362 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
6363 isa<InsertValueInst>(it))
6364 PostProcessInstructions.push_back(&*it);
6370 bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
6371 auto Changed = false;
6372 for (auto &Entry : GEPs) {
6373 // If the getelementptr list has fewer than two elements, there's nothing
6375 if (Entry.second.size() < 2)
6378 LLVM_DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "
6379 << Entry.second.size() << ".\n");
6381 // We process the getelementptr list in chunks of 16 (like we do for
6382 // stores) to minimize compile-time.
6383 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
6384 auto Len = std::min<unsigned>(BE - BI, 16);
6385 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
6387 // Initialize a set a candidate getelementptrs. Note that we use a
6388 // SetVector here to preserve program order. If the index computations
6389 // are vectorizable and begin with loads, we want to minimize the chance
6390 // of having to reorder them later.
6391 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
6393 // Some of the candidates may have already been vectorized after we
6394 // initially collected them. If so, the WeakTrackingVHs will have
6396 // values, so remove them from the set of candidates.
6397 Candidates.remove(nullptr);
6399 // Remove from the set of candidates all pairs of getelementptrs with
6400 // constant differences. Such getelementptrs are likely not good
6401 // candidates for vectorization in a bottom-up phase since one can be
6402 // computed from the other. We also ensure all candidate getelementptr
6403 // indices are unique.
6404 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
6405 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
6406 if (!Candidates.count(GEPI))
6408 auto *SCEVI = SE->getSCEV(GEPList[I]);
6409 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
6410 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
6411 auto *SCEVJ = SE->getSCEV(GEPList[J]);
6412 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
6413 Candidates.remove(GEPList[I]);
6414 Candidates.remove(GEPList[J]);
6415 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
6416 Candidates.remove(GEPList[J]);
6421 // We break out of the above computation as soon as we know there are
6422 // fewer than two candidates remaining.
6423 if (Candidates.size() < 2)
6426 // Add the single, non-constant index of each candidate to the bundle. We
6427 // ensured the indices met these constraints when we originally collected
6428 // the getelementptrs.
6429 SmallVector<Value *, 16> Bundle(Candidates.size());
6430 auto BundleIndex = 0u;
6431 for (auto *V : Candidates) {
6432 auto *GEP = cast<GetElementPtrInst>(V);
6433 auto *GEPIdx = GEP->idx_begin()->get();
6434 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx));
6435 Bundle[BundleIndex++] = GEPIdx;
6438 // Try and vectorize the indices. We are currently only interested in
6439 // gather-like cases of the form:
6441 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
6443 // where the loads of "a", the loads of "b", and the subtractions can be
6444 // performed in parallel. It's likely that detecting this pattern in a
6445 // bottom-up phase will be simpler and less costly than building a
6446 // full-blown top-down phase beginning at the consecutive loads.
6447 Changed |= tryToVectorizeList(Bundle, R);
6453 bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
6454 bool Changed = false;
6455 // Attempt to sort and vectorize each of the store-groups.
6456 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
6458 if (it->second.size() < 2)
6461 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length "
6462 << it->second.size() << ".\n");
6464 // Process the stores in chunks of 16.
6465 // TODO: The limit of 16 inhibits greater vectorization factors.
6466 // For example, AVX2 supports v32i8. Increasing this limit, however,
6467 // may cause a significant compile-time increase.
6468 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI += 16) {
6469 unsigned Len = std::min<unsigned>(CE - CI, 16);
6470 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
6476 char SLPVectorizer::ID = 0;
6478 static const char lv_name[] = "SLP Vectorizer";
6480 INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)
6481 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
6482 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
6483 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6484 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
6485 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
6486 INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)
6487 INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
6488 INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)
6490 Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); }