1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11 // stores that can be put together into vector-stores. Next, it attempts to
12 // construct vectorizable tree using the use-def chains. If a profitable tree
13 // was found, the SLP vectorizer performs vectorization on the tree.
15 // The pass is inspired by the work described in the paper:
16 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
18 //===----------------------------------------------------------------------===//
20 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/DenseSet.h"
24 #include "llvm/ADT/MapVector.h"
25 #include "llvm/ADT/None.h"
26 #include "llvm/ADT/Optional.h"
27 #include "llvm/ADT/PostOrderIterator.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SetVector.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/SmallSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/Statistic.h"
34 #include "llvm/ADT/iterator.h"
35 #include "llvm/ADT/iterator_range.h"
36 #include "llvm/Analysis/AliasAnalysis.h"
37 #include "llvm/Analysis/CodeMetrics.h"
38 #include "llvm/Analysis/DemandedBits.h"
39 #include "llvm/Analysis/GlobalsModRef.h"
40 #include "llvm/Analysis/LoopAccessAnalysis.h"
41 #include "llvm/Analysis/LoopInfo.h"
42 #include "llvm/Analysis/MemoryLocation.h"
43 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
44 #include "llvm/Analysis/ScalarEvolution.h"
45 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
46 #include "llvm/Analysis/TargetLibraryInfo.h"
47 #include "llvm/Analysis/TargetTransformInfo.h"
48 #include "llvm/Analysis/ValueTracking.h"
49 #include "llvm/Analysis/VectorUtils.h"
50 #include "llvm/IR/Attributes.h"
51 #include "llvm/IR/BasicBlock.h"
52 #include "llvm/IR/Constant.h"
53 #include "llvm/IR/Constants.h"
54 #include "llvm/IR/DataLayout.h"
55 #include "llvm/IR/DebugLoc.h"
56 #include "llvm/IR/DerivedTypes.h"
57 #include "llvm/IR/Dominators.h"
58 #include "llvm/IR/Function.h"
59 #include "llvm/IR/IRBuilder.h"
60 #include "llvm/IR/InstrTypes.h"
61 #include "llvm/IR/Instruction.h"
62 #include "llvm/IR/Instructions.h"
63 #include "llvm/IR/IntrinsicInst.h"
64 #include "llvm/IR/Intrinsics.h"
65 #include "llvm/IR/Module.h"
66 #include "llvm/IR/NoFolder.h"
67 #include "llvm/IR/Operator.h"
68 #include "llvm/IR/PassManager.h"
69 #include "llvm/IR/PatternMatch.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/Pass.h"
77 #include "llvm/Support/Casting.h"
78 #include "llvm/Support/CommandLine.h"
79 #include "llvm/Support/Compiler.h"
80 #include "llvm/Support/DOTGraphTraits.h"
81 #include "llvm/Support/Debug.h"
82 #include "llvm/Support/ErrorHandling.h"
83 #include "llvm/Support/GraphWriter.h"
84 #include "llvm/Support/KnownBits.h"
85 #include "llvm/Support/MathExtras.h"
86 #include "llvm/Support/raw_ostream.h"
87 #include "llvm/Transforms/Utils/LoopUtils.h"
88 #include "llvm/Transforms/Vectorize.h"
100 using namespace llvm;
101 using namespace llvm::PatternMatch;
102 using namespace slpvectorizer;
104 #define SV_NAME "slp-vectorizer"
105 #define DEBUG_TYPE "SLP"
107 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
115 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
118 static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
121 "Attempt to vectorize horizontal reductions feeding into a store"));
124 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
127 /// Limits the size of scheduling regions in a block.
128 /// It avoid long compile times for _very_ large blocks where vector
129 /// instructions are spread over a wide range.
130 /// This limit is way higher than needed by real-world functions.
132 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
135 static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
139 static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
143 static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
151 // Limit the number of alias checks. The limit is chosen so that
152 // it has no negative effect on the llvm benchmarks.
153 static const unsigned AliasedCheckLimit = 10;
155 // Another limit for the alias checks: The maximum distance between load/store
156 // instructions where alias checks are done.
157 // This limit is useful for very large basic blocks.
158 static const unsigned MaxMemDepDistance = 160;
160 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161 /// regions to be handled.
162 static const int MinScheduleRegionSize = 16;
164 /// \brief Predicate for the element types that the SLP vectorizer supports.
166 /// The most important thing to filter here are types which are invalid in LLVM
167 /// vectors. We also filter target specific types which have absolutely no
168 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169 /// avoids spending time checking the cost model and realizing that they will
170 /// be inevitably scalarized.
171 static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
176 /// \returns true if all of the instructions in \p VL are in the same block or
178 static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
188 if (BB != I->getParent())
194 /// \returns True if all of the values in \p VL are constants.
195 static bool allConstant(ArrayRef<Value *> VL) {
197 if (!isa<Constant>(i))
202 /// \returns True if all of the values in \p VL are identical.
203 static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
210 /// Checks if the vector of instructions can be represented as a shuffle, like:
211 /// %x0 = extractelement <4 x i8> %x, i32 0
212 /// %x3 = extractelement <4 x i8> %x, i32 3
213 /// %y1 = extractelement <4 x i8> %y, i32 1
214 /// %y2 = extractelement <4 x i8> %y, i32 2
215 /// %x0x0 = mul i8 %x0, %x0
216 /// %x3x3 = mul i8 %x3, %x3
217 /// %y1y1 = mul i8 %y1, %y1
218 /// %y2y2 = mul i8 %y2, %y2
219 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223 /// ret <4 x i8> %ins4
224 /// can be transformed into:
225 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
227 /// %2 = mul <4 x i8> %1, %1
229 /// We convert this initially to something like:
230 /// %x0 = extractelement <4 x i8> %x, i32 0
231 /// %x3 = extractelement <4 x i8> %x, i32 3
232 /// %y1 = extractelement <4 x i8> %y, i32 1
233 /// %y2 = extractelement <4 x i8> %y, i32 2
234 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238 /// %5 = mul <4 x i8> %4, %4
239 /// %6 = extractelement <4 x i8> %5, i32 0
240 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241 /// %7 = extractelement <4 x i8> %5, i32 1
242 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243 /// %8 = extractelement <4 x i8> %5, i32 2
244 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245 /// %9 = extractelement <4 x i8> %5, i32 3
246 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247 /// ret <4 x i8> %ins4
248 /// InstCombiner transforms this into a shuffle and vector mul
249 static Optional<TargetTransformInfo::ShuffleKind>
250 isShuffle(ArrayRef<Value *> VL) {
251 auto *EI0 = cast<ExtractElementInst>(VL[0]);
252 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253 Value *Vec1 = nullptr;
254 Value *Vec2 = nullptr;
255 enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256 ShuffleMode CommonShuffleMode = Unknown;
257 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258 auto *EI = cast<ExtractElementInst>(VL[I]);
259 auto *Vec = EI->getVectorOperand();
260 // All vector operands must have the same number of vector elements.
261 if (Vec->getType()->getVectorNumElements() != Size)
263 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
266 // Undefined behavior if Idx is negative or >= Size.
267 if (Idx->getValue().uge(Size))
269 unsigned IntIdx = Idx->getValue().getZExtValue();
270 // We can extractelement from undef vector.
271 if (isa<UndefValue>(Vec))
273 // For correct shuffling we have to have at most 2 different vector operands
274 // in all extractelement instructions.
275 if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
277 if (CommonShuffleMode == Permute)
279 // If the extract index is not the same as the operation number, it is a
282 CommonShuffleMode = Permute;
285 // Check the shuffle mode for the current operation.
288 else if (Vec != Vec1)
290 // Example: shufflevector A, B, <0,5,2,7>
291 // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292 // I is even and IntIdx for B == I - FirstAlternate shuffle.
293 // Example: shufflevector A, B, <4,1,6,3>
294 // I is even and IntIdx for A == I - SecondAlternate shuffle.
295 // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296 const bool IIsEven = I & 1;
297 const bool CurrVecIsA = Vec == Vec1;
298 const bool IIsOdd = !IIsEven;
299 const bool CurrVecIsB = !CurrVecIsA;
300 ShuffleMode CurrentShuffleMode =
301 ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
303 // Common mode is not set or the same as the shuffle mode of the current
304 // operation - alternate.
305 if (CommonShuffleMode == Unknown)
306 CommonShuffleMode = CurrentShuffleMode;
307 // Common shuffle mode is not the same as the shuffle mode of the current
308 // operation - permutation.
309 if (CommonShuffleMode != CurrentShuffleMode)
310 CommonShuffleMode = Permute;
312 // If we're not crossing lanes in different vectors, consider it as blending.
313 if ((CommonShuffleMode == FirstAlternate ||
314 CommonShuffleMode == SecondAlternate) &&
316 return TargetTransformInfo::SK_Alternate;
317 // If Vec2 was never used, we have a permutation of a single vector, otherwise
318 // we have permutation of 2 vectors.
319 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
320 : TargetTransformInfo::SK_PermuteSingleSrc;
323 ///\returns Opcode that can be clubbed with \p Op to create an alternate
324 /// sequence which can later be merged as a ShuffleVector instruction.
325 static unsigned getAltOpcode(unsigned Op) {
327 case Instruction::FAdd:
328 return Instruction::FSub;
329 case Instruction::FSub:
330 return Instruction::FAdd;
331 case Instruction::Add:
332 return Instruction::Sub;
333 case Instruction::Sub:
334 return Instruction::Add;
340 static bool isOdd(unsigned Value) {
344 static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345 unsigned CheckedOpcode) {
346 return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
349 /// Chooses the correct key for scheduling data. If \p Op has the same (or
350 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
352 static Value *isOneOf(Value *OpValue, Value *Op) {
353 auto *I = dyn_cast<Instruction>(Op);
356 auto *OpInst = cast<Instruction>(OpValue);
357 unsigned OpInstOpcode = OpInst->getOpcode();
358 unsigned IOpcode = I->getOpcode();
359 if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
366 /// Contains data for the instructions going to be vectorized.
367 struct RawInstructionsData {
368 /// Main Opcode of the instructions going to be vectorized.
371 /// The list of instructions have some instructions with alternate opcodes.
372 bool HasAltOpcodes = false;
375 } // end anonymous namespace
377 /// Checks the list of the vectorized instructions \p VL and returns info about
379 static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380 auto *I0 = dyn_cast<Instruction>(VL[0]);
383 RawInstructionsData Res;
384 unsigned Opcode = I0->getOpcode();
385 // Walk through the list of the vectorized instructions
386 // in order to check its structure described by RawInstructionsData.
387 for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388 auto *I = dyn_cast<Instruction>(VL[Cnt]);
391 if (Opcode != I->getOpcode())
392 Res.HasAltOpcodes = true;
400 /// Main data required for vectorization of instructions.
401 struct InstructionsState {
402 /// The very first instruction in the list with the main opcode.
403 Value *OpValue = nullptr;
405 /// The main opcode for the list of instructions.
408 /// Some of the instructions in the list have alternate opcodes.
409 bool IsAltShuffle = false;
411 InstructionsState() = default;
412 InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413 : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
416 } // end anonymous namespace
418 /// \returns analysis of the Instructions in \p VL described in
419 /// InstructionsState, the Opcode that we suppose the whole list
420 /// could be vectorized even if its structure is diverse.
421 static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422 auto Res = getMainOpcode(VL);
423 unsigned Opcode = Res.Opcode;
424 if (!Res.HasAltOpcodes)
425 return InstructionsState(VL[0], Opcode, false);
426 auto *OpInst = cast<Instruction>(VL[0]);
427 unsigned AltOpcode = getAltOpcode(Opcode);
428 // Examine each element in the list instructions VL to determine
429 // if some operations there could be considered as an alternative
430 // (for example as subtraction relates to addition operation).
431 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432 auto *I = cast<Instruction>(VL[Cnt]);
433 unsigned InstOpcode = I->getOpcode();
434 if ((Res.HasAltOpcodes &&
435 InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436 (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437 return InstructionsState(OpInst, 0, false);
440 return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
443 /// \returns true if all of the values in \p VL have the same type or false
445 static bool allSameType(ArrayRef<Value *> VL) {
446 Type *Ty = VL[0]->getType();
447 for (int i = 1, e = VL.size(); i < e; i++)
448 if (VL[i]->getType() != Ty)
454 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
455 static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
456 assert(Opcode == Instruction::ExtractElement ||
457 Opcode == Instruction::ExtractValue);
458 if (Opcode == Instruction::ExtractElement) {
459 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
460 return CI && CI->getZExtValue() == Idx;
462 ExtractValueInst *EI = cast<ExtractValueInst>(E);
463 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
467 /// \returns True if in-tree use also needs extract. This refers to
468 /// possible scalar operand in vectorized instruction.
469 static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
470 TargetLibraryInfo *TLI) {
471 unsigned Opcode = UserInst->getOpcode();
473 case Instruction::Load: {
474 LoadInst *LI = cast<LoadInst>(UserInst);
475 return (LI->getPointerOperand() == Scalar);
477 case Instruction::Store: {
478 StoreInst *SI = cast<StoreInst>(UserInst);
479 return (SI->getPointerOperand() == Scalar);
481 case Instruction::Call: {
482 CallInst *CI = cast<CallInst>(UserInst);
483 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
484 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
485 return (CI->getArgOperand(1) == Scalar);
494 /// \returns the AA location that is being access by the instruction.
495 static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
496 if (StoreInst *SI = dyn_cast<StoreInst>(I))
497 return MemoryLocation::get(SI);
498 if (LoadInst *LI = dyn_cast<LoadInst>(I))
499 return MemoryLocation::get(LI);
500 return MemoryLocation();
503 /// \returns True if the instruction is not a volatile or atomic load/store.
504 static bool isSimple(Instruction *I) {
505 if (LoadInst *LI = dyn_cast<LoadInst>(I))
506 return LI->isSimple();
507 if (StoreInst *SI = dyn_cast<StoreInst>(I))
508 return SI->isSimple();
509 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
510 return !MI->isVolatile();
516 namespace slpvectorizer {
518 /// Bottom Up SLP Vectorizer.
521 using ValueList = SmallVector<Value *, 8>;
522 using InstrList = SmallVector<Instruction *, 16>;
523 using ValueSet = SmallPtrSet<Value *, 16>;
524 using StoreList = SmallVector<StoreInst *, 8>;
525 using ExtraValueToDebugLocsMap =
526 MapVector<Value *, SmallVector<Instruction *, 2>>;
528 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
529 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
530 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
531 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
532 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
533 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
534 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
535 // Use the vector register size specified by the target unless overridden
536 // by a command-line option.
537 // TODO: It would be better to limit the vectorization factor based on
538 // data type rather than just register size. For example, x86 AVX has
539 // 256-bit registers, but it does not support integer operations
540 // at that width (that requires AVX2).
541 if (MaxVectorRegSizeOption.getNumOccurrences())
542 MaxVecRegSize = MaxVectorRegSizeOption;
544 MaxVecRegSize = TTI->getRegisterBitWidth(true);
546 if (MinVectorRegSizeOption.getNumOccurrences())
547 MinVecRegSize = MinVectorRegSizeOption;
549 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
552 /// \brief Vectorize the tree that starts with the elements in \p VL.
553 /// Returns the vectorized root.
554 Value *vectorizeTree();
556 /// Vectorize the tree but with the list of externally used values \p
557 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
558 /// generated extractvalue instructions.
559 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
561 /// \returns the cost incurred by unwanted spills and fills, caused by
562 /// holding live values over call sites.
565 /// \returns the vectorization cost of the subtree that starts at \p VL.
566 /// A negative number means that this is profitable.
569 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
570 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
571 void buildTree(ArrayRef<Value *> Roots,
572 ArrayRef<Value *> UserIgnoreLst = None);
574 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
576 /// into account (anf updating it, if required) list of externally used
577 /// values stored in \p ExternallyUsedValues.
578 void buildTree(ArrayRef<Value *> Roots,
579 ExtraValueToDebugLocsMap &ExternallyUsedValues,
580 ArrayRef<Value *> UserIgnoreLst = None);
582 /// Clear the internal data structures that are created by 'buildTree'.
584 VectorizableTree.clear();
585 ScalarToTreeEntry.clear();
587 ExternalUses.clear();
588 NumLoadsWantToKeepOrder = 0;
589 NumLoadsWantToChangeOrder = 0;
590 for (auto &Iter : BlocksSchedules) {
591 BlockScheduling *BS = Iter.second.get();
597 unsigned getTreeSize() const { return VectorizableTree.size(); }
599 /// \brief Perform LICM and CSE on the newly generated gather sequences.
600 void optimizeGatherSequence(Function &F);
602 /// \returns true if it is beneficial to reverse the vector order.
603 bool shouldReorder() const {
604 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
607 /// \return The vector element size in bits to use when vectorizing the
608 /// expression tree ending at \p V. If V is a store, the size is the width of
609 /// the stored value. Otherwise, the size is the width of the largest loaded
610 /// value reaching V. This method is used by the vectorizer to calculate
611 /// vectorization factors.
612 unsigned getVectorElementSize(Value *V);
614 /// Compute the minimum type sizes required to represent the entries in a
615 /// vectorizable tree.
616 void computeMinimumValueSizes();
618 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
619 unsigned getMaxVecRegSize() const {
620 return MaxVecRegSize;
623 // \returns minimum vector register size as set by cl::opt.
624 unsigned getMinVecRegSize() const {
625 return MinVecRegSize;
628 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
630 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
631 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
633 /// \returns True if the VectorizableTree is both tiny and not fully
634 /// vectorizable. We do not vectorize such trees.
635 bool isTreeTinyAndNotFullyVectorizable();
637 OptimizationRemarkEmitter *getORE() { return ORE; }
642 /// Checks if all users of \p I are the part of the vectorization tree.
643 bool areAllUsersVectorized(Instruction *I) const;
645 /// \returns the cost of the vectorizable entry.
646 int getEntryCost(TreeEntry *E);
648 /// This is the recursive part of buildTree.
649 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
651 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
652 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
653 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const;
655 /// Vectorize a single entry in the tree.
656 Value *vectorizeTree(TreeEntry *E);
658 /// Vectorize a single entry in the tree, starting in \p VL.
659 Value *vectorizeTree(ArrayRef<Value *> VL);
661 /// \returns the pointer to the vectorized value if \p VL is already
662 /// vectorized, or NULL. They may happen in cycles.
663 Value *alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const;
665 /// \returns the scalarization cost for this type. Scalarization in this
666 /// context means the creation of vectors from a group of scalars.
667 int getGatherCost(Type *Ty);
669 /// \returns the scalarization cost for this list of values. Assuming that
670 /// this subtree gets vectorized, we may need to extract the values from the
671 /// roots. This method calculates the cost of extracting the values.
672 int getGatherCost(ArrayRef<Value *> VL);
674 /// \brief Set the Builder insert point to one after the last instruction in
676 void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
678 /// \returns a vector from a collection of scalars in \p VL.
679 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
681 /// \returns whether the VectorizableTree is fully vectorizable and will
682 /// be beneficial even the tree height is tiny.
683 bool isFullyVectorizableTinyTree();
685 /// \reorder commutative operands in alt shuffle if they result in
687 void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
688 SmallVectorImpl<Value *> &Left,
689 SmallVectorImpl<Value *> &Right);
691 /// \reorder commutative operands to get better probability of
692 /// generating vectorized code.
693 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
694 SmallVectorImpl<Value *> &Left,
695 SmallVectorImpl<Value *> &Right);
697 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
699 /// \returns true if the scalars in VL are equal to this entry.
700 bool isSame(ArrayRef<Value *> VL) const {
701 assert(VL.size() == Scalars.size() && "Invalid size");
702 return std::equal(VL.begin(), VL.end(), Scalars.begin());
705 /// A vector of scalars.
708 /// The Scalars are vectorized into this value. It is initialized to Null.
709 Value *VectorizedValue = nullptr;
711 /// Do we need to gather this sequence ?
712 bool NeedToGather = false;
714 /// Points back to the VectorizableTree.
716 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
717 /// to be a pointer and needs to be able to initialize the child iterator.
718 /// Thus we need a reference back to the container to translate the indices
720 std::vector<TreeEntry> &Container;
722 /// The TreeEntry index containing the user of this entry. We can actually
723 /// have multiple users so the data structure is not truly a tree.
724 SmallVector<int, 1> UserTreeIndices;
727 /// Create a new VectorizableTree entry.
728 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
730 VectorizableTree.emplace_back(VectorizableTree);
731 int idx = VectorizableTree.size() - 1;
732 TreeEntry *Last = &VectorizableTree[idx];
733 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
734 Last->NeedToGather = !Vectorized;
736 for (int i = 0, e = VL.size(); i != e; ++i) {
737 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
738 ScalarToTreeEntry[VL[i]] = idx;
741 MustGather.insert(VL.begin(), VL.end());
744 if (UserTreeIdx >= 0)
745 Last->UserTreeIndices.push_back(UserTreeIdx);
750 /// -- Vectorization State --
751 /// Holds all of the tree entries.
752 std::vector<TreeEntry> VectorizableTree;
754 TreeEntry *getTreeEntry(Value *V) {
755 auto I = ScalarToTreeEntry.find(V);
756 if (I != ScalarToTreeEntry.end())
757 return &VectorizableTree[I->second];
761 const TreeEntry *getTreeEntry(Value *V) const {
762 auto I = ScalarToTreeEntry.find(V);
763 if (I != ScalarToTreeEntry.end())
764 return &VectorizableTree[I->second];
768 /// Maps a specific scalar to its tree entry.
769 SmallDenseMap<Value*, int> ScalarToTreeEntry;
771 /// A list of scalars that we found that we need to keep as scalars.
774 /// This POD struct describes one external user in the vectorized tree.
775 struct ExternalUser {
776 ExternalUser(Value *S, llvm::User *U, int L)
777 : Scalar(S), User(U), Lane(L) {}
779 // Which scalar in our function.
782 // Which user that uses the scalar.
785 // Which lane does the scalar belong to.
788 using UserList = SmallVector<ExternalUser, 16>;
790 /// Checks if two instructions may access the same memory.
792 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
793 /// is invariant in the calling loop.
794 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
795 Instruction *Inst2) {
796 // First check if the result is already in the cache.
797 AliasCacheKey key = std::make_pair(Inst1, Inst2);
798 Optional<bool> &result = AliasCache[key];
799 if (result.hasValue()) {
800 return result.getValue();
802 MemoryLocation Loc2 = getLocation(Inst2, AA);
804 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
805 // Do the alias check.
806 aliased = AA->alias(Loc1, Loc2);
808 // Store the result in the cache.
813 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
815 /// Cache for alias results.
816 /// TODO: consider moving this to the AliasAnalysis itself.
817 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
819 /// Removes an instruction from its block and eventually deletes it.
820 /// It's like Instruction::eraseFromParent() except that the actual deletion
821 /// is delayed until BoUpSLP is destructed.
822 /// This is required to ensure that there are no incorrect collisions in the
823 /// AliasCache, which can happen if a new instruction is allocated at the
824 /// same address as a previously deleted instruction.
825 void eraseInstruction(Instruction *I) {
826 I->removeFromParent();
827 I->dropAllReferences();
828 DeletedInstructions.emplace_back(I);
831 /// Temporary store for deleted instructions. Instructions will be deleted
832 /// eventually when the BoUpSLP is destructed.
833 SmallVector<unique_value, 8> DeletedInstructions;
835 /// A list of values that need to extracted out of the tree.
836 /// This list holds pairs of (Internal Scalar : External User). External User
837 /// can be nullptr, it means that this Internal Scalar will be used later,
838 /// after vectorization.
839 UserList ExternalUses;
841 /// Values used only by @llvm.assume calls.
842 SmallPtrSet<const Value *, 32> EphValues;
844 /// Holds all of the instructions that we gathered.
845 SetVector<Instruction *> GatherSeq;
847 /// A list of blocks that we are going to CSE.
848 SetVector<BasicBlock *> CSEBlocks;
850 /// Contains all scheduling relevant data for an instruction.
851 /// A ScheduleData either represents a single instruction or a member of an
852 /// instruction bundle (= a group of instructions which is combined into a
853 /// vector instruction).
854 struct ScheduleData {
855 // The initial value for the dependency counters. It means that the
856 // dependencies are not calculated yet.
857 enum { InvalidDeps = -1 };
859 ScheduleData() = default;
861 void init(int BlockSchedulingRegionID, Value *OpVal) {
862 FirstInBundle = this;
863 NextInBundle = nullptr;
864 NextLoadStore = nullptr;
866 SchedulingRegionID = BlockSchedulingRegionID;
867 UnscheduledDepsInBundle = UnscheduledDeps;
872 /// Returns true if the dependency information has been calculated.
873 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
875 /// Returns true for single instructions and for bundle representatives
876 /// (= the head of a bundle).
877 bool isSchedulingEntity() const { return FirstInBundle == this; }
879 /// Returns true if it represents an instruction bundle and not only a
880 /// single instruction.
881 bool isPartOfBundle() const {
882 return NextInBundle != nullptr || FirstInBundle != this;
885 /// Returns true if it is ready for scheduling, i.e. it has no more
886 /// unscheduled depending instructions/bundles.
887 bool isReady() const {
888 assert(isSchedulingEntity() &&
889 "can't consider non-scheduling entity for ready list");
890 return UnscheduledDepsInBundle == 0 && !IsScheduled;
893 /// Modifies the number of unscheduled dependencies, also updating it for
894 /// the whole bundle.
895 int incrementUnscheduledDeps(int Incr) {
896 UnscheduledDeps += Incr;
897 return FirstInBundle->UnscheduledDepsInBundle += Incr;
900 /// Sets the number of unscheduled dependencies to the number of
902 void resetUnscheduledDeps() {
903 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
906 /// Clears all dependency information.
907 void clearDependencies() {
908 Dependencies = InvalidDeps;
909 resetUnscheduledDeps();
910 MemoryDependencies.clear();
913 void dump(raw_ostream &os) const {
914 if (!isSchedulingEntity()) {
916 } else if (NextInBundle) {
918 ScheduleData *SD = NextInBundle;
920 os << ';' << *SD->Inst;
921 SD = SD->NextInBundle;
929 Instruction *Inst = nullptr;
931 /// Points to the head in an instruction bundle (and always to this for
932 /// single instructions).
933 ScheduleData *FirstInBundle = nullptr;
935 /// Single linked list of all instructions in a bundle. Null if it is a
936 /// single instruction.
937 ScheduleData *NextInBundle = nullptr;
939 /// Single linked list of all memory instructions (e.g. load, store, call)
940 /// in the block - until the end of the scheduling region.
941 ScheduleData *NextLoadStore = nullptr;
943 /// The dependent memory instructions.
944 /// This list is derived on demand in calculateDependencies().
945 SmallVector<ScheduleData *, 4> MemoryDependencies;
947 /// This ScheduleData is in the current scheduling region if this matches
948 /// the current SchedulingRegionID of BlockScheduling.
949 int SchedulingRegionID = 0;
951 /// Used for getting a "good" final ordering of instructions.
952 int SchedulingPriority = 0;
954 /// The number of dependencies. Constitutes of the number of users of the
955 /// instruction plus the number of dependent memory instructions (if any).
956 /// This value is calculated on demand.
957 /// If InvalidDeps, the number of dependencies is not calculated yet.
958 int Dependencies = InvalidDeps;
960 /// The number of dependencies minus the number of dependencies of scheduled
961 /// instructions. As soon as this is zero, the instruction/bundle gets ready
963 /// Note that this is negative as long as Dependencies is not calculated.
964 int UnscheduledDeps = InvalidDeps;
966 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
967 /// single instructions.
968 int UnscheduledDepsInBundle = InvalidDeps;
970 /// True if this instruction is scheduled (or considered as scheduled in the
972 bool IsScheduled = false;
974 /// Opcode of the current instruction in the schedule data.
975 Value *OpValue = nullptr;
979 friend inline raw_ostream &operator<<(raw_ostream &os,
980 const BoUpSLP::ScheduleData &SD) {
986 friend struct GraphTraits<BoUpSLP *>;
987 friend struct DOTGraphTraits<BoUpSLP *>;
989 /// Contains all scheduling data for a basic block.
990 struct BlockScheduling {
991 BlockScheduling(BasicBlock *BB)
992 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
996 ScheduleStart = nullptr;
997 ScheduleEnd = nullptr;
998 FirstLoadStoreInRegion = nullptr;
999 LastLoadStoreInRegion = nullptr;
1001 // Reduce the maximum schedule region size by the size of the
1002 // previous scheduling run.
1003 ScheduleRegionSizeLimit -= ScheduleRegionSize;
1004 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1005 ScheduleRegionSizeLimit = MinScheduleRegionSize;
1006 ScheduleRegionSize = 0;
1008 // Make a new scheduling region, i.e. all existing ScheduleData is not
1009 // in the new region yet.
1010 ++SchedulingRegionID;
1013 ScheduleData *getScheduleData(Value *V) {
1014 ScheduleData *SD = ScheduleDataMap[V];
1015 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1020 ScheduleData *getScheduleData(Value *V, Value *Key) {
1022 return getScheduleData(V);
1023 auto I = ExtraScheduleDataMap.find(V);
1024 if (I != ExtraScheduleDataMap.end()) {
1025 ScheduleData *SD = I->second[Key];
1026 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1032 bool isInSchedulingRegion(ScheduleData *SD) {
1033 return SD->SchedulingRegionID == SchedulingRegionID;
1036 /// Marks an instruction as scheduled and puts all dependent ready
1037 /// instructions into the ready-list.
1038 template <typename ReadyListType>
1039 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1040 SD->IsScheduled = true;
1041 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1043 ScheduleData *BundleMember = SD;
1044 while (BundleMember) {
1045 if (BundleMember->Inst != BundleMember->OpValue) {
1046 BundleMember = BundleMember->NextInBundle;
1049 // Handle the def-use chain dependencies.
1050 for (Use &U : BundleMember->Inst->operands()) {
1051 auto *I = dyn_cast<Instruction>(U.get());
1054 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1055 if (OpDef && OpDef->hasValidDependencies() &&
1056 OpDef->incrementUnscheduledDeps(-1) == 0) {
1057 // There are no more unscheduled dependencies after
1058 // decrementing, so we can put the dependent instruction
1059 // into the ready list.
1060 ScheduleData *DepBundle = OpDef->FirstInBundle;
1061 assert(!DepBundle->IsScheduled &&
1062 "already scheduled bundle gets ready");
1063 ReadyList.insert(DepBundle);
1065 << "SLP: gets ready (def): " << *DepBundle << "\n");
1069 // Handle the memory dependencies.
1070 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1071 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1072 // There are no more unscheduled dependencies after decrementing,
1073 // so we can put the dependent instruction into the ready list.
1074 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1075 assert(!DepBundle->IsScheduled &&
1076 "already scheduled bundle gets ready");
1077 ReadyList.insert(DepBundle);
1078 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle
1082 BundleMember = BundleMember->NextInBundle;
1086 void doForAllOpcodes(Value *V,
1087 function_ref<void(ScheduleData *SD)> Action) {
1088 if (ScheduleData *SD = getScheduleData(V))
1090 auto I = ExtraScheduleDataMap.find(V);
1091 if (I != ExtraScheduleDataMap.end())
1092 for (auto &P : I->second)
1093 if (P.second->SchedulingRegionID == SchedulingRegionID)
1097 /// Put all instructions into the ReadyList which are ready for scheduling.
1098 template <typename ReadyListType>
1099 void initialFillReadyList(ReadyListType &ReadyList) {
1100 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1101 doForAllOpcodes(I, [&](ScheduleData *SD) {
1102 if (SD->isSchedulingEntity() && SD->isReady()) {
1103 ReadyList.insert(SD);
1104 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n");
1110 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1111 /// cyclic dependencies. This is only a dry-run, no instructions are
1112 /// actually moved at this stage.
1113 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1115 /// Un-bundles a group of instructions.
1116 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1118 /// Allocates schedule data chunk.
1119 ScheduleData *allocateScheduleDataChunks();
1121 /// Extends the scheduling region so that V is inside the region.
1122 /// \returns true if the region size is within the limit.
1123 bool extendSchedulingRegion(Value *V, Value *OpValue);
1125 /// Initialize the ScheduleData structures for new instructions in the
1126 /// scheduling region.
1127 void initScheduleData(Instruction *FromI, Instruction *ToI,
1128 ScheduleData *PrevLoadStore,
1129 ScheduleData *NextLoadStore);
1131 /// Updates the dependency information of a bundle and of all instructions/
1132 /// bundles which depend on the original bundle.
1133 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1136 /// Sets all instruction in the scheduling region to un-scheduled.
1137 void resetSchedule();
1141 /// Simple memory allocation for ScheduleData.
1142 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1144 /// The size of a ScheduleData array in ScheduleDataChunks.
1147 /// The allocator position in the current chunk, which is the last entry
1148 /// of ScheduleDataChunks.
1151 /// Attaches ScheduleData to Instruction.
1152 /// Note that the mapping survives during all vectorization iterations, i.e.
1153 /// ScheduleData structures are recycled.
1154 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1156 /// Attaches ScheduleData to Instruction with the leading key.
1157 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1158 ExtraScheduleDataMap;
1160 struct ReadyList : SmallVector<ScheduleData *, 8> {
1161 void insert(ScheduleData *SD) { push_back(SD); }
1164 /// The ready-list for scheduling (only used for the dry-run).
1165 ReadyList ReadyInsts;
1167 /// The first instruction of the scheduling region.
1168 Instruction *ScheduleStart = nullptr;
1170 /// The first instruction _after_ the scheduling region.
1171 Instruction *ScheduleEnd = nullptr;
1173 /// The first memory accessing instruction in the scheduling region
1175 ScheduleData *FirstLoadStoreInRegion = nullptr;
1177 /// The last memory accessing instruction in the scheduling region
1179 ScheduleData *LastLoadStoreInRegion = nullptr;
1181 /// The current size of the scheduling region.
1182 int ScheduleRegionSize = 0;
1184 /// The maximum size allowed for the scheduling region.
1185 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1187 /// The ID of the scheduling region. For a new vectorization iteration this
1188 /// is incremented which "removes" all ScheduleData from the region.
1189 // Make sure that the initial SchedulingRegionID is greater than the
1190 // initial SchedulingRegionID in ScheduleData (which is 0).
1191 int SchedulingRegionID = 1;
1194 /// Attaches the BlockScheduling structures to basic blocks.
1195 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1197 /// Performs the "real" scheduling. Done before vectorization is actually
1198 /// performed in a basic block.
1199 void scheduleBlock(BlockScheduling *BS);
1201 /// List of users to ignore during scheduling and that don't need extracting.
1202 ArrayRef<Value *> UserIgnoreList;
1204 // Number of load bundles that contain consecutive loads.
1205 int NumLoadsWantToKeepOrder = 0;
1207 // Number of load bundles that contain consecutive loads in reversed order.
1208 int NumLoadsWantToChangeOrder = 0;
1210 // Analysis and block reference.
1212 ScalarEvolution *SE;
1213 TargetTransformInfo *TTI;
1214 TargetLibraryInfo *TLI;
1218 AssumptionCache *AC;
1220 const DataLayout *DL;
1221 OptimizationRemarkEmitter *ORE;
1223 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1224 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1226 /// Instruction builder to construct the vectorized tree.
1227 IRBuilder<> Builder;
1229 /// A map of scalar integer values to the smallest bit width with which they
1230 /// can legally be represented. The values map to (width, signed) pairs,
1231 /// where "width" indicates the minimum bit width and "signed" is True if the
1232 /// value must be signed-extended, rather than zero-extended, back to its
1234 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1237 } // end namespace slpvectorizer
1239 template <> struct GraphTraits<BoUpSLP *> {
1240 using TreeEntry = BoUpSLP::TreeEntry;
1242 /// NodeRef has to be a pointer per the GraphWriter.
1243 using NodeRef = TreeEntry *;
1245 /// \brief Add the VectorizableTree to the index iterator to be able to return
1246 /// TreeEntry pointers.
1247 struct ChildIteratorType
1248 : public iterator_adaptor_base<ChildIteratorType,
1249 SmallVector<int, 1>::iterator> {
1250 std::vector<TreeEntry> &VectorizableTree;
1252 ChildIteratorType(SmallVector<int, 1>::iterator W,
1253 std::vector<TreeEntry> &VT)
1254 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1256 NodeRef operator*() { return &VectorizableTree[*I]; }
1259 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1261 static ChildIteratorType child_begin(NodeRef N) {
1262 return {N->UserTreeIndices.begin(), N->Container};
1265 static ChildIteratorType child_end(NodeRef N) {
1266 return {N->UserTreeIndices.end(), N->Container};
1269 /// For the node iterator we just need to turn the TreeEntry iterator into a
1270 /// TreeEntry* iterator so that it dereferences to NodeRef.
1271 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1273 static nodes_iterator nodes_begin(BoUpSLP *R) {
1274 return nodes_iterator(R->VectorizableTree.begin());
1277 static nodes_iterator nodes_end(BoUpSLP *R) {
1278 return nodes_iterator(R->VectorizableTree.end());
1281 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1284 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1285 using TreeEntry = BoUpSLP::TreeEntry;
1287 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1289 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1291 raw_string_ostream OS(Str);
1292 if (isSplat(Entry->Scalars)) {
1293 OS << "<splat> " << *Entry->Scalars[0];
1296 for (auto V : Entry->Scalars) {
1299 R->ExternalUses.begin(), R->ExternalUses.end(),
1300 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1307 static std::string getNodeAttributes(const TreeEntry *Entry,
1309 if (Entry->NeedToGather)
1315 } // end namespace llvm
1317 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1318 ArrayRef<Value *> UserIgnoreLst) {
1319 ExtraValueToDebugLocsMap ExternallyUsedValues;
1320 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1323 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1324 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1325 ArrayRef<Value *> UserIgnoreLst) {
1327 UserIgnoreList = UserIgnoreLst;
1328 if (!allSameType(Roots))
1330 buildTree_rec(Roots, 0, -1);
1332 // Collect the values that we need to extract from the tree.
1333 for (TreeEntry &EIdx : VectorizableTree) {
1334 TreeEntry *Entry = &EIdx;
1336 // No need to handle users of gathered values.
1337 if (Entry->NeedToGather)
1341 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1342 Value *Scalar = Entry->Scalars[Lane];
1344 // Check if the scalar is externally used as an extra arg.
1345 auto ExtI = ExternallyUsedValues.find(Scalar);
1346 if (ExtI != ExternallyUsedValues.end()) {
1347 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<
1348 Lane << " from " << *Scalar << ".\n");
1349 ExternalUses.emplace_back(Scalar, nullptr, Lane);
1352 for (User *U : Scalar->users()) {
1353 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1355 Instruction *UserInst = dyn_cast<Instruction>(U);
1359 // Skip in-tree scalars that become vectors
1360 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1361 Value *UseScalar = UseEntry->Scalars[0];
1362 // Some in-tree scalars will remain as scalar in vectorized
1363 // instructions. If that is the case, the one in Lane 0 will
1365 if (UseScalar != U ||
1366 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1367 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1369 assert(!UseEntry->NeedToGather && "Bad state");
1374 // Ignore users in the user ignore list.
1375 if (is_contained(UserIgnoreList, UserInst))
1378 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<
1379 Lane << " from " << *Scalar << ".\n");
1380 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1386 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1388 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1390 InstructionsState S = getSameOpcode(VL);
1391 if (Depth == RecursionMaxDepth) {
1392 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1393 newTreeEntry(VL, false, UserTreeIdx);
1397 // Don't handle vectors.
1398 if (S.OpValue->getType()->isVectorTy()) {
1399 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1400 newTreeEntry(VL, false, UserTreeIdx);
1404 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1405 if (SI->getValueOperand()->getType()->isVectorTy()) {
1406 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1407 newTreeEntry(VL, false, UserTreeIdx);
1411 // If all of the operands are identical or constant we have a simple solution.
1412 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1413 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1414 newTreeEntry(VL, false, UserTreeIdx);
1418 // We now know that this is a vector of instructions of the same type from
1421 // Don't vectorize ephemeral values.
1422 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1423 if (EphValues.count(VL[i])) {
1424 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1425 ") is ephemeral.\n");
1426 newTreeEntry(VL, false, UserTreeIdx);
1431 // Check if this is a duplicate of another entry.
1432 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1433 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1434 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n");
1435 if (E->Scalars[i] != VL[i]) {
1436 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1437 newTreeEntry(VL, false, UserTreeIdx);
1441 // Record the reuse of the tree node. FIXME, currently this is only used to
1442 // properly draw the graph rather than for the actual vectorization.
1443 E->UserTreeIndices.push_back(UserTreeIdx);
1444 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n");
1448 // Check that none of the instructions in the bundle are already in the tree.
1449 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1450 auto *I = dyn_cast<Instruction>(VL[i]);
1453 if (getTreeEntry(I)) {
1454 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1455 ") is already in tree.\n");
1456 newTreeEntry(VL, false, UserTreeIdx);
1461 // If any of the scalars is marked as a value that needs to stay scalar, then
1462 // we need to gather the scalars.
1463 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1464 if (MustGather.count(VL[i])) {
1465 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1466 newTreeEntry(VL, false, UserTreeIdx);
1471 // Check that all of the users of the scalars that we want to vectorize are
1473 auto *VL0 = cast<Instruction>(S.OpValue);
1474 BasicBlock *BB = VL0->getParent();
1476 if (!DT->isReachableFromEntry(BB)) {
1477 // Don't go into unreachable blocks. They may contain instructions with
1478 // dependency cycles which confuse the final scheduling.
1479 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1480 newTreeEntry(VL, false, UserTreeIdx);
1484 // Check that every instruction appears once in this bundle.
1485 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1486 for (unsigned j = i + 1; j < e; ++j)
1487 if (VL[i] == VL[j]) {
1488 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1489 newTreeEntry(VL, false, UserTreeIdx);
1493 auto &BSRef = BlocksSchedules[BB];
1495 BSRef = llvm::make_unique<BlockScheduling>(BB);
1497 BlockScheduling &BS = *BSRef.get();
1499 if (!BS.tryScheduleBundle(VL, this, S.OpValue)) {
1500 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1501 assert((!BS.getScheduleData(VL0) ||
1502 !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1503 "tryScheduleBundle should cancelScheduling on failure");
1504 newTreeEntry(VL, false, UserTreeIdx);
1507 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1509 unsigned ShuffleOrOp = S.IsAltShuffle ?
1510 (unsigned) Instruction::ShuffleVector : S.Opcode;
1511 switch (ShuffleOrOp) {
1512 case Instruction::PHI: {
1513 PHINode *PH = dyn_cast<PHINode>(VL0);
1515 // Check for terminator values (e.g. invoke).
1516 for (unsigned j = 0; j < VL.size(); ++j)
1517 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1518 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1519 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1521 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n");
1522 BS.cancelScheduling(VL, VL0);
1523 newTreeEntry(VL, false, UserTreeIdx);
1528 newTreeEntry(VL, true, UserTreeIdx);
1529 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1531 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1533 // Prepare the operand vector.
1535 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1536 PH->getIncomingBlock(i)));
1538 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1542 case Instruction::ExtractValue:
1543 case Instruction::ExtractElement: {
1544 bool Reuse = canReuseExtract(VL, VL0);
1546 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n");
1548 BS.cancelScheduling(VL, VL0);
1550 newTreeEntry(VL, Reuse, UserTreeIdx);
1553 case Instruction::Load: {
1554 // Check that a vectorized load would load the same memory as a scalar
1555 // load. For example, we don't want to vectorize loads that are smaller
1556 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1557 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1558 // from such a struct, we read/write packed bits disagreeing with the
1559 // unvectorized version.
1560 Type *ScalarTy = VL0->getType();
1562 if (DL->getTypeSizeInBits(ScalarTy) !=
1563 DL->getTypeAllocSizeInBits(ScalarTy)) {
1564 BS.cancelScheduling(VL, VL0);
1565 newTreeEntry(VL, false, UserTreeIdx);
1566 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1570 // Make sure all loads in the bundle are simple - we can't vectorize
1571 // atomic or volatile loads.
1572 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1573 LoadInst *L = cast<LoadInst>(VL[i]);
1574 if (!L->isSimple()) {
1575 BS.cancelScheduling(VL, VL0);
1576 newTreeEntry(VL, false, UserTreeIdx);
1577 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1582 // Check if the loads are consecutive, reversed, or neither.
1583 // TODO: What we really want is to sort the loads, but for now, check
1584 // the two likely directions.
1585 bool Consecutive = true;
1586 bool ReverseConsecutive = true;
1587 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1588 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1589 Consecutive = false;
1592 ReverseConsecutive = false;
1597 ++NumLoadsWantToKeepOrder;
1598 newTreeEntry(VL, true, UserTreeIdx);
1599 DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1603 // If none of the load pairs were consecutive when checked in order,
1604 // check the reverse order.
1605 if (ReverseConsecutive)
1606 for (unsigned i = VL.size() - 1; i > 0; --i)
1607 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1608 ReverseConsecutive = false;
1612 BS.cancelScheduling(VL, VL0);
1613 newTreeEntry(VL, false, UserTreeIdx);
1615 if (ReverseConsecutive) {
1616 ++NumLoadsWantToChangeOrder;
1617 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n");
1619 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1623 case Instruction::ZExt:
1624 case Instruction::SExt:
1625 case Instruction::FPToUI:
1626 case Instruction::FPToSI:
1627 case Instruction::FPExt:
1628 case Instruction::PtrToInt:
1629 case Instruction::IntToPtr:
1630 case Instruction::SIToFP:
1631 case Instruction::UIToFP:
1632 case Instruction::Trunc:
1633 case Instruction::FPTrunc:
1634 case Instruction::BitCast: {
1635 Type *SrcTy = VL0->getOperand(0)->getType();
1636 for (unsigned i = 0; i < VL.size(); ++i) {
1637 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1638 if (Ty != SrcTy || !isValidElementType(Ty)) {
1639 BS.cancelScheduling(VL, VL0);
1640 newTreeEntry(VL, false, UserTreeIdx);
1641 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n");
1645 newTreeEntry(VL, true, UserTreeIdx);
1646 DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1648 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1650 // Prepare the operand vector.
1652 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1654 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1658 case Instruction::ICmp:
1659 case Instruction::FCmp: {
1660 // Check that all of the compares have the same predicate.
1661 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1662 Type *ComparedTy = VL0->getOperand(0)->getType();
1663 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1664 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1665 if (Cmp->getPredicate() != P0 ||
1666 Cmp->getOperand(0)->getType() != ComparedTy) {
1667 BS.cancelScheduling(VL, VL0);
1668 newTreeEntry(VL, false, UserTreeIdx);
1669 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
1674 newTreeEntry(VL, true, UserTreeIdx);
1675 DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1677 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1679 // Prepare the operand vector.
1681 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1683 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1687 case Instruction::Select:
1688 case Instruction::Add:
1689 case Instruction::FAdd:
1690 case Instruction::Sub:
1691 case Instruction::FSub:
1692 case Instruction::Mul:
1693 case Instruction::FMul:
1694 case Instruction::UDiv:
1695 case Instruction::SDiv:
1696 case Instruction::FDiv:
1697 case Instruction::URem:
1698 case Instruction::SRem:
1699 case Instruction::FRem:
1700 case Instruction::Shl:
1701 case Instruction::LShr:
1702 case Instruction::AShr:
1703 case Instruction::And:
1704 case Instruction::Or:
1705 case Instruction::Xor:
1706 newTreeEntry(VL, true, UserTreeIdx);
1707 DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1709 // Sort operands of the instructions so that each side is more likely to
1710 // have the same opcode.
1711 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1712 ValueList Left, Right;
1713 reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1714 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1715 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1719 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1721 // Prepare the operand vector.
1723 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1725 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1729 case Instruction::GetElementPtr: {
1730 // We don't combine GEPs with complicated (nested) indexing.
1731 for (unsigned j = 0; j < VL.size(); ++j) {
1732 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1733 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1734 BS.cancelScheduling(VL, VL0);
1735 newTreeEntry(VL, false, UserTreeIdx);
1740 // We can't combine several GEPs into one vector if they operate on
1742 Type *Ty0 = VL0->getOperand(0)->getType();
1743 for (unsigned j = 0; j < VL.size(); ++j) {
1744 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1746 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
1747 BS.cancelScheduling(VL, VL0);
1748 newTreeEntry(VL, false, UserTreeIdx);
1753 // We don't combine GEPs with non-constant indexes.
1754 for (unsigned j = 0; j < VL.size(); ++j) {
1755 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1756 if (!isa<ConstantInt>(Op)) {
1758 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1759 BS.cancelScheduling(VL, VL0);
1760 newTreeEntry(VL, false, UserTreeIdx);
1765 newTreeEntry(VL, true, UserTreeIdx);
1766 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1767 for (unsigned i = 0, e = 2; i < e; ++i) {
1769 // Prepare the operand vector.
1771 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1773 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1777 case Instruction::Store: {
1778 // Check if the stores are consecutive or of we need to swizzle them.
1779 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1780 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1781 BS.cancelScheduling(VL, VL0);
1782 newTreeEntry(VL, false, UserTreeIdx);
1783 DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1787 newTreeEntry(VL, true, UserTreeIdx);
1788 DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1792 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1794 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1797 case Instruction::Call: {
1798 // Check if the calls are all to the same vectorizable intrinsic.
1799 CallInst *CI = cast<CallInst>(VL0);
1800 // Check if this is an Intrinsic call or something that can be
1801 // represented by an intrinsic call
1802 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1803 if (!isTriviallyVectorizable(ID)) {
1804 BS.cancelScheduling(VL, VL0);
1805 newTreeEntry(VL, false, UserTreeIdx);
1806 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1809 Function *Int = CI->getCalledFunction();
1810 Value *A1I = nullptr;
1811 if (hasVectorInstrinsicScalarOpd(ID, 1))
1812 A1I = CI->getArgOperand(1);
1813 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1814 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1815 if (!CI2 || CI2->getCalledFunction() != Int ||
1816 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1817 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1818 BS.cancelScheduling(VL, VL0);
1819 newTreeEntry(VL, false, UserTreeIdx);
1820 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1824 // ctlz,cttz and powi are special intrinsics whose second argument
1825 // should be same in order for them to be vectorized.
1826 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1827 Value *A1J = CI2->getArgOperand(1);
1829 BS.cancelScheduling(VL, VL0);
1830 newTreeEntry(VL, false, UserTreeIdx);
1831 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1832 << " argument "<< A1I<<"!=" << A1J
1837 // Verify that the bundle operands are identical between the two calls.
1838 if (CI->hasOperandBundles() &&
1839 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1840 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1841 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1842 BS.cancelScheduling(VL, VL0);
1843 newTreeEntry(VL, false, UserTreeIdx);
1844 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="
1850 newTreeEntry(VL, true, UserTreeIdx);
1851 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1853 // Prepare the operand vector.
1854 for (Value *j : VL) {
1855 CallInst *CI2 = dyn_cast<CallInst>(j);
1856 Operands.push_back(CI2->getArgOperand(i));
1858 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1862 case Instruction::ShuffleVector:
1863 // If this is not an alternate sequence of opcode like add-sub
1864 // then do not vectorize this instruction.
1865 if (!S.IsAltShuffle) {
1866 BS.cancelScheduling(VL, VL0);
1867 newTreeEntry(VL, false, UserTreeIdx);
1868 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1871 newTreeEntry(VL, true, UserTreeIdx);
1872 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1874 // Reorder operands if reordering would enable vectorization.
1875 if (isa<BinaryOperator>(VL0)) {
1876 ValueList Left, Right;
1877 reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1878 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1879 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1883 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1885 // Prepare the operand vector.
1887 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1889 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1894 BS.cancelScheduling(VL, VL0);
1895 newTreeEntry(VL, false, UserTreeIdx);
1896 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1901 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1904 auto *ST = dyn_cast<StructType>(T);
1906 N = ST->getNumElements();
1907 EltTy = *ST->element_begin();
1909 N = cast<ArrayType>(T)->getNumElements();
1910 EltTy = cast<ArrayType>(T)->getElementType();
1912 if (!isValidElementType(EltTy))
1914 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1915 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1918 // Check that struct is homogeneous.
1919 for (const auto *Ty : ST->elements())
1926 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const {
1927 Instruction *E0 = cast<Instruction>(OpValue);
1928 assert(E0->getOpcode() == Instruction::ExtractElement ||
1929 E0->getOpcode() == Instruction::ExtractValue);
1930 assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode");
1931 // Check if all of the extracts come from the same vector and from the
1933 Value *Vec = E0->getOperand(0);
1935 // We have to extract from a vector/aggregate with the same number of elements.
1937 if (E0->getOpcode() == Instruction::ExtractValue) {
1938 const DataLayout &DL = E0->getModule()->getDataLayout();
1939 NElts = canMapToVector(Vec->getType(), DL);
1942 // Check if load can be rewritten as load of vector.
1943 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1944 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1947 NElts = Vec->getType()->getVectorNumElements();
1950 if (NElts != VL.size())
1953 // Check that all of the indices extract from the correct offset.
1954 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
1955 Instruction *Inst = cast<Instruction>(VL[I]);
1956 if (!matchExtractIndex(Inst, I, Inst->getOpcode()))
1958 if (Inst->getOperand(0) != Vec)
1965 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
1966 return I->hasOneUse() ||
1967 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
1968 return ScalarToTreeEntry.count(U) > 0;
1972 int BoUpSLP::getEntryCost(TreeEntry *E) {
1973 ArrayRef<Value*> VL = E->Scalars;
1975 Type *ScalarTy = VL[0]->getType();
1976 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1977 ScalarTy = SI->getValueOperand()->getType();
1978 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
1979 ScalarTy = CI->getOperand(0)->getType();
1980 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1982 // If we have computed a smaller type for the expression, update VecTy so
1983 // that the costs will be accurate.
1984 if (MinBWs.count(VL[0]))
1985 VecTy = VectorType::get(
1986 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1988 if (E->NeedToGather) {
1989 if (allConstant(VL))
1992 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1994 if (getSameOpcode(VL).Opcode == Instruction::ExtractElement) {
1995 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
1996 if (ShuffleKind.hasValue()) {
1997 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
1998 for (auto *V : VL) {
1999 // If all users of instruction are going to be vectorized and this
2000 // instruction itself is not going to be vectorized, consider this
2001 // instruction as dead and remove its cost from the final cost of the
2003 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2004 !ScalarToTreeEntry.count(V)) {
2005 auto *IO = cast<ConstantInt>(
2006 cast<ExtractElementInst>(V)->getIndexOperand());
2007 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2008 IO->getZExtValue());
2014 return getGatherCost(E->Scalars);
2016 InstructionsState S = getSameOpcode(VL);
2017 assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2018 Instruction *VL0 = cast<Instruction>(S.OpValue);
2019 unsigned ShuffleOrOp = S.IsAltShuffle ?
2020 (unsigned) Instruction::ShuffleVector : S.Opcode;
2021 switch (ShuffleOrOp) {
2022 case Instruction::PHI:
2025 case Instruction::ExtractValue:
2026 case Instruction::ExtractElement:
2027 if (canReuseExtract(VL, S.OpValue)) {
2029 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2030 Instruction *E = cast<Instruction>(VL[i]);
2031 // If all users are going to be vectorized, instruction can be
2032 // considered as dead.
2033 // The same, if have only one user, it will be vectorized for sure.
2034 if (areAllUsersVectorized(E))
2035 // Take credit for instruction that will become dead.
2037 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2041 return getGatherCost(VecTy);
2043 case Instruction::ZExt:
2044 case Instruction::SExt:
2045 case Instruction::FPToUI:
2046 case Instruction::FPToSI:
2047 case Instruction::FPExt:
2048 case Instruction::PtrToInt:
2049 case Instruction::IntToPtr:
2050 case Instruction::SIToFP:
2051 case Instruction::UIToFP:
2052 case Instruction::Trunc:
2053 case Instruction::FPTrunc:
2054 case Instruction::BitCast: {
2055 Type *SrcTy = VL0->getOperand(0)->getType();
2057 // Calculate the cost of this instruction.
2058 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2059 VL0->getType(), SrcTy, VL0);
2061 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2062 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2063 return VecCost - ScalarCost;
2065 case Instruction::FCmp:
2066 case Instruction::ICmp:
2067 case Instruction::Select: {
2068 // Calculate the cost of this instruction.
2069 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2070 int ScalarCost = VecTy->getNumElements() *
2071 TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2072 int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2073 return VecCost - ScalarCost;
2075 case Instruction::Add:
2076 case Instruction::FAdd:
2077 case Instruction::Sub:
2078 case Instruction::FSub:
2079 case Instruction::Mul:
2080 case Instruction::FMul:
2081 case Instruction::UDiv:
2082 case Instruction::SDiv:
2083 case Instruction::FDiv:
2084 case Instruction::URem:
2085 case Instruction::SRem:
2086 case Instruction::FRem:
2087 case Instruction::Shl:
2088 case Instruction::LShr:
2089 case Instruction::AShr:
2090 case Instruction::And:
2091 case Instruction::Or:
2092 case Instruction::Xor: {
2093 // Certain instructions can be cheaper to vectorize if they have a
2094 // constant second vector operand.
2095 TargetTransformInfo::OperandValueKind Op1VK =
2096 TargetTransformInfo::OK_AnyValue;
2097 TargetTransformInfo::OperandValueKind Op2VK =
2098 TargetTransformInfo::OK_UniformConstantValue;
2099 TargetTransformInfo::OperandValueProperties Op1VP =
2100 TargetTransformInfo::OP_None;
2101 TargetTransformInfo::OperandValueProperties Op2VP =
2102 TargetTransformInfo::OP_None;
2104 // If all operands are exactly the same ConstantInt then set the
2105 // operand kind to OK_UniformConstantValue.
2106 // If instead not all operands are constants, then set the operand kind
2107 // to OK_AnyValue. If all operands are constants but not the same,
2108 // then set the operand kind to OK_NonUniformConstantValue.
2109 ConstantInt *CInt = nullptr;
2110 for (unsigned i = 0; i < VL.size(); ++i) {
2111 const Instruction *I = cast<Instruction>(VL[i]);
2112 if (!isa<ConstantInt>(I->getOperand(1))) {
2113 Op2VK = TargetTransformInfo::OK_AnyValue;
2117 CInt = cast<ConstantInt>(I->getOperand(1));
2120 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
2121 CInt != cast<ConstantInt>(I->getOperand(1)))
2122 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2124 // FIXME: Currently cost of model modification for division by power of
2125 // 2 is handled for X86 and AArch64. Add support for other targets.
2126 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2127 CInt->getValue().isPowerOf2())
2128 Op2VP = TargetTransformInfo::OP_PowerOf2;
2130 SmallVector<const Value *, 4> Operands(VL0->operand_values());
2132 VecTy->getNumElements() *
2133 TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2135 int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2136 Op1VP, Op2VP, Operands);
2137 return VecCost - ScalarCost;
2139 case Instruction::GetElementPtr: {
2140 TargetTransformInfo::OperandValueKind Op1VK =
2141 TargetTransformInfo::OK_AnyValue;
2142 TargetTransformInfo::OperandValueKind Op2VK =
2143 TargetTransformInfo::OK_UniformConstantValue;
2146 VecTy->getNumElements() *
2147 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2149 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2151 return VecCost - ScalarCost;
2153 case Instruction::Load: {
2154 // Cost of wide load - cost of scalar loads.
2155 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2156 int ScalarLdCost = VecTy->getNumElements() *
2157 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2158 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2159 VecTy, alignment, 0, VL0);
2160 return VecLdCost - ScalarLdCost;
2162 case Instruction::Store: {
2163 // We know that we can merge the stores. Calculate the cost.
2164 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2165 int ScalarStCost = VecTy->getNumElements() *
2166 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2167 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2168 VecTy, alignment, 0, VL0);
2169 return VecStCost - ScalarStCost;
2171 case Instruction::Call: {
2172 CallInst *CI = cast<CallInst>(VL0);
2173 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2175 // Calculate the cost of the scalar and vector calls.
2176 SmallVector<Type*, 4> ScalarTys;
2177 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2178 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2181 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2182 FMF = FPMO->getFastMathFlags();
2184 int ScalarCallCost = VecTy->getNumElements() *
2185 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2187 SmallVector<Value *, 4> Args(CI->arg_operands());
2188 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2189 VecTy->getNumElements());
2191 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCost
2192 << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2193 << " for " << *CI << "\n");
2195 return VecCallCost - ScalarCallCost;
2197 case Instruction::ShuffleVector: {
2198 TargetTransformInfo::OperandValueKind Op1VK =
2199 TargetTransformInfo::OK_AnyValue;
2200 TargetTransformInfo::OperandValueKind Op2VK =
2201 TargetTransformInfo::OK_AnyValue;
2204 for (Value *i : VL) {
2205 Instruction *I = cast<Instruction>(i);
2209 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2211 // VecCost is equal to sum of the cost of creating 2 vectors
2212 // and the cost of creating shuffle.
2213 Instruction *I0 = cast<Instruction>(VL[0]);
2215 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2216 Instruction *I1 = cast<Instruction>(VL[1]);
2218 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2220 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2221 return VecCost - ScalarCost;
2224 llvm_unreachable("Unknown instruction");
2228 bool BoUpSLP::isFullyVectorizableTinyTree() {
2229 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<
2230 VectorizableTree.size() << " is fully vectorizable .\n");
2232 // We only handle trees of heights 1 and 2.
2233 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2236 if (VectorizableTree.size() != 2)
2239 // Handle splat and all-constants stores.
2240 if (!VectorizableTree[0].NeedToGather &&
2241 (allConstant(VectorizableTree[1].Scalars) ||
2242 isSplat(VectorizableTree[1].Scalars)))
2245 // Gathering cost would be too much for tiny trees.
2246 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2252 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2253 // We can vectorize the tree if its size is greater than or equal to the
2254 // minimum size specified by the MinTreeSize command line option.
2255 if (VectorizableTree.size() >= MinTreeSize)
2258 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2259 // can vectorize it if we can prove it fully vectorizable.
2260 if (isFullyVectorizableTinyTree())
2263 assert(VectorizableTree.empty()
2264 ? ExternalUses.empty()
2265 : true && "We shouldn't have any external users");
2267 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2272 int BoUpSLP::getSpillCost() {
2273 // Walk from the bottom of the tree to the top, tracking which values are
2274 // live. When we see a call instruction that is not part of our tree,
2275 // query TTI to see if there is a cost to keeping values live over it
2276 // (for example, if spills and fills are required).
2277 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2280 SmallPtrSet<Instruction*, 4> LiveValues;
2281 Instruction *PrevInst = nullptr;
2283 for (const auto &N : VectorizableTree) {
2284 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2293 // Update LiveValues.
2294 LiveValues.erase(PrevInst);
2295 for (auto &J : PrevInst->operands()) {
2296 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2297 LiveValues.insert(cast<Instruction>(&*J));
2301 dbgs() << "SLP: #LV: " << LiveValues.size();
2302 for (auto *X : LiveValues)
2303 dbgs() << " " << X->getName();
2304 dbgs() << ", Looking at ";
2308 // Now find the sequence of instructions between PrevInst and Inst.
2309 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2311 PrevInst->getIterator().getReverse();
2312 while (InstIt != PrevInstIt) {
2313 if (PrevInstIt == PrevInst->getParent()->rend()) {
2314 PrevInstIt = Inst->getParent()->rbegin();
2318 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2319 SmallVector<Type*, 4> V;
2320 for (auto *II : LiveValues)
2321 V.push_back(VectorType::get(II->getType(), BundleWidth));
2322 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2334 int BoUpSLP::getTreeCost() {
2336 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<
2337 VectorizableTree.size() << ".\n");
2339 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2341 for (TreeEntry &TE : VectorizableTree) {
2342 int C = getEntryCost(&TE);
2343 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "
2344 << *TE.Scalars[0] << ".\n");
2348 SmallSet<Value *, 16> ExtractCostCalculated;
2349 int ExtractCost = 0;
2350 for (ExternalUser &EU : ExternalUses) {
2351 // We only add extract cost once for the same scalar.
2352 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2355 // Uses by ephemeral values are free (because the ephemeral value will be
2356 // removed prior to code generation, and so the extraction will be
2357 // removed as well).
2358 if (EphValues.count(EU.User))
2361 // If we plan to rewrite the tree in a smaller type, we will need to sign
2362 // extend the extracted value back to the original type. Here, we account
2363 // for the extract and the added cost of the sign extend if needed.
2364 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2365 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2366 if (MinBWs.count(ScalarRoot)) {
2367 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2369 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2370 VecTy = VectorType::get(MinTy, BundleWidth);
2371 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2375 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2379 int SpillCost = getSpillCost();
2380 Cost += SpillCost + ExtractCost;
2384 raw_string_ostream OS(Str);
2385 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2386 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2387 << "SLP: Total Cost = " << Cost << ".\n";
2389 DEBUG(dbgs() << Str);
2392 ViewGraph(this, "SLP" + F->getName(), false, Str);
2397 int BoUpSLP::getGatherCost(Type *Ty) {
2399 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2400 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2404 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2405 // Find the type of the operands in VL.
2406 Type *ScalarTy = VL[0]->getType();
2407 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2408 ScalarTy = SI->getValueOperand()->getType();
2409 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2410 // Find the cost of inserting/extracting values from the vector.
2411 return getGatherCost(VecTy);
2414 // Reorder commutative operations in alternate shuffle if the resulting vectors
2415 // are consecutive loads. This would allow us to vectorize the tree.
2416 // If we have something like-
2417 // load a[0] - load b[0]
2418 // load b[1] + load a[1]
2419 // load a[2] - load b[2]
2420 // load a[3] + load b[3]
2421 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2423 void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2424 SmallVectorImpl<Value *> &Left,
2425 SmallVectorImpl<Value *> &Right) {
2426 // Push left and right operands of binary operation into Left and Right
2427 unsigned AltOpcode = getAltOpcode(Opcode);
2429 for (Value *V : VL) {
2430 auto *I = cast<Instruction>(V);
2431 assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&
2432 "Incorrect instruction in vector");
2433 Left.push_back(I->getOperand(0));
2434 Right.push_back(I->getOperand(1));
2437 // Reorder if we have a commutative operation and consecutive access
2438 // are on either side of the alternate instructions.
2439 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2440 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2441 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2442 Instruction *VL1 = cast<Instruction>(VL[j]);
2443 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2444 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2445 std::swap(Left[j], Right[j]);
2447 } else if (VL2->isCommutative() &&
2448 isConsecutiveAccess(L, L1, *DL, *SE)) {
2449 std::swap(Left[j + 1], Right[j + 1]);
2455 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2456 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2457 Instruction *VL1 = cast<Instruction>(VL[j]);
2458 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2459 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2460 std::swap(Left[j], Right[j]);
2462 } else if (VL2->isCommutative() &&
2463 isConsecutiveAccess(L, L1, *DL, *SE)) {
2464 std::swap(Left[j + 1], Right[j + 1]);
2473 // Return true if I should be commuted before adding it's left and right
2474 // operands to the arrays Left and Right.
2476 // The vectorizer is trying to either have all elements one side being
2477 // instruction with the same opcode to enable further vectorization, or having
2478 // a splat to lower the vectorizing cost.
2479 static bool shouldReorderOperands(
2480 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2481 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2482 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2483 VLeft = I.getOperand(0);
2484 VRight = I.getOperand(1);
2485 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2487 if (VRight == Right[i - 1])
2488 // Preserve SplatRight
2490 if (VLeft == Right[i - 1]) {
2491 // Commuting would preserve SplatRight, but we don't want to break
2492 // SplatLeft either, i.e. preserve the original order if possible.
2493 // (FIXME: why do we care?)
2494 if (SplatLeft && VLeft == Left[i - 1])
2499 // Symmetrically handle Right side.
2501 if (VLeft == Left[i - 1])
2502 // Preserve SplatLeft
2504 if (VRight == Left[i - 1])
2508 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2509 Instruction *IRight = dyn_cast<Instruction>(VRight);
2511 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2512 // it and not the right, in this case we want to commute.
2513 if (AllSameOpcodeRight) {
2514 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2515 if (IRight && RightPrevOpcode == IRight->getOpcode())
2516 // Do not commute, a match on the right preserves AllSameOpcodeRight
2518 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2519 // We have a match and may want to commute, but first check if there is
2520 // not also a match on the existing operands on the Left to preserve
2521 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2522 // (FIXME: why do we care?)
2523 if (AllSameOpcodeLeft && ILeft &&
2524 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2529 // Symmetrically handle Left side.
2530 if (AllSameOpcodeLeft) {
2531 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2532 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2534 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2540 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2541 ArrayRef<Value *> VL,
2542 SmallVectorImpl<Value *> &Left,
2543 SmallVectorImpl<Value *> &Right) {
2545 // Peel the first iteration out of the loop since there's nothing
2546 // interesting to do anyway and it simplifies the checks in the loop.
2547 auto *I = cast<Instruction>(VL[0]);
2548 Value *VLeft = I->getOperand(0);
2549 Value *VRight = I->getOperand(1);
2550 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2551 // Favor having instruction to the right. FIXME: why?
2552 std::swap(VLeft, VRight);
2553 Left.push_back(VLeft);
2554 Right.push_back(VRight);
2557 // Keep track if we have instructions with all the same opcode on one side.
2558 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2559 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2560 // Keep track if we have one side with all the same value (broadcast).
2561 bool SplatLeft = true;
2562 bool SplatRight = true;
2564 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2565 Instruction *I = cast<Instruction>(VL[i]);
2566 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2567 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2568 "Can only process commutative instruction");
2569 // Commute to favor either a splat or maximizing having the same opcodes on
2573 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2574 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2576 Left.push_back(VRight);
2577 Right.push_back(VLeft);
2579 Left.push_back(VLeft);
2580 Right.push_back(VRight);
2582 // Update Splat* and AllSameOpcode* after the insertion.
2583 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2584 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2585 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2586 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2587 cast<Instruction>(Left[i])->getOpcode());
2588 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2589 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2590 cast<Instruction>(Right[i])->getOpcode());
2593 // If one operand end up being broadcast, return this operand order.
2594 if (SplatRight || SplatLeft)
2597 // Finally check if we can get longer vectorizable chain by reordering
2598 // without breaking the good operand order detected above.
2599 // E.g. If we have something like-
2600 // load a[0] load b[0]
2601 // load b[1] load a[1]
2602 // load a[2] load b[2]
2603 // load a[3] load b[3]
2604 // Reordering the second load b[1] load a[1] would allow us to vectorize
2605 // this code and we still retain AllSameOpcode property.
2606 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2608 // add a[0],c[0] load b[0]
2609 // add a[1],c[2] load b[1]
2611 // add a[3],c[3] load b[3]
2612 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2613 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2614 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2615 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2616 std::swap(Left[j + 1], Right[j + 1]);
2621 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2622 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2623 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2624 std::swap(Left[j + 1], Right[j + 1]);
2633 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2634 // Get the basic block this bundle is in. All instructions in the bundle
2635 // should be in this block.
2636 auto *Front = cast<Instruction>(OpValue);
2637 auto *BB = Front->getParent();
2638 const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2639 const unsigned AltOpcode = getAltOpcode(Opcode);
2640 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2641 return !sameOpcodeOrAlt(Opcode, AltOpcode,
2642 cast<Instruction>(V)->getOpcode()) ||
2643 cast<Instruction>(V)->getParent() == BB;
2646 // The last instruction in the bundle in program order.
2647 Instruction *LastInst = nullptr;
2649 // Find the last instruction. The common case should be that BB has been
2650 // scheduled, and the last instruction is VL.back(). So we start with
2651 // VL.back() and iterate over schedule data until we reach the end of the
2652 // bundle. The end of the bundle is marked by null ScheduleData.
2653 if (BlocksSchedules.count(BB)) {
2655 BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2656 if (Bundle && Bundle->isPartOfBundle())
2657 for (; Bundle; Bundle = Bundle->NextInBundle)
2658 if (Bundle->OpValue == Bundle->Inst)
2659 LastInst = Bundle->Inst;
2662 // LastInst can still be null at this point if there's either not an entry
2663 // for BB in BlocksSchedules or there's no ScheduleData available for
2664 // VL.back(). This can be the case if buildTree_rec aborts for various
2665 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2666 // size is reached, etc.). ScheduleData is initialized in the scheduling
2669 // If this happens, we can still find the last instruction by brute force. We
2670 // iterate forwards from Front (inclusive) until we either see all
2671 // instructions in the bundle or reach the end of the block. If Front is the
2672 // last instruction in program order, LastInst will be set to Front, and we
2673 // will visit all the remaining instructions in the block.
2675 // One of the reasons we exit early from buildTree_rec is to place an upper
2676 // bound on compile-time. Thus, taking an additional compile-time hit here is
2677 // not ideal. However, this should be exceedingly rare since it requires that
2678 // we both exit early from buildTree_rec and that the bundle be out-of-order
2679 // (causing us to iterate all the way to the end of the block).
2681 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2682 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2683 if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2690 // Set the insertion point after the last instruction in the bundle. Set the
2691 // debug location to Front.
2692 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2693 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2696 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2697 Value *Vec = UndefValue::get(Ty);
2698 // Generate the 'InsertElement' instruction.
2699 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2700 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2701 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2702 GatherSeq.insert(Insrt);
2703 CSEBlocks.insert(Insrt->getParent());
2705 // Add to our 'need-to-extract' list.
2706 if (TreeEntry *E = getTreeEntry(VL[i])) {
2707 // Find which lane we need to extract.
2709 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2710 // Is this the lane of the scalar that we are looking for ?
2711 if (E->Scalars[Lane] == VL[i]) {
2716 assert(FoundLane >= 0 && "Could not find the correct lane");
2717 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2725 Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const {
2726 if (const TreeEntry *En = getTreeEntry(OpValue)) {
2727 if (En->isSame(VL) && En->VectorizedValue)
2728 return En->VectorizedValue;
2733 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2734 InstructionsState S = getSameOpcode(VL);
2736 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2738 return vectorizeTree(E);
2742 Type *ScalarTy = S.OpValue->getType();
2743 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2744 ScalarTy = SI->getValueOperand()->getType();
2745 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2747 return Gather(VL, VecTy);
2750 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2751 IRBuilder<>::InsertPointGuard Guard(Builder);
2753 if (E->VectorizedValue) {
2754 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
2755 return E->VectorizedValue;
2758 InstructionsState S = getSameOpcode(E->Scalars);
2759 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2760 Type *ScalarTy = VL0->getType();
2761 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2762 ScalarTy = SI->getValueOperand()->getType();
2763 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2765 if (E->NeedToGather) {
2766 setInsertPointAfterBundle(E->Scalars, VL0);
2767 auto *V = Gather(E->Scalars, VecTy);
2768 E->VectorizedValue = V;
2772 unsigned ShuffleOrOp = S.IsAltShuffle ?
2773 (unsigned) Instruction::ShuffleVector : S.Opcode;
2774 switch (ShuffleOrOp) {
2775 case Instruction::PHI: {
2776 PHINode *PH = dyn_cast<PHINode>(VL0);
2777 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2778 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2779 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2780 E->VectorizedValue = NewPhi;
2782 // PHINodes may have multiple entries from the same block. We want to
2783 // visit every block once.
2784 SmallSet<BasicBlock*, 4> VisitedBBs;
2786 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2788 BasicBlock *IBB = PH->getIncomingBlock(i);
2790 if (!VisitedBBs.insert(IBB).second) {
2791 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2795 // Prepare the operand vector.
2796 for (Value *V : E->Scalars)
2797 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2799 Builder.SetInsertPoint(IBB->getTerminator());
2800 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2801 Value *Vec = vectorizeTree(Operands);
2802 NewPhi->addIncoming(Vec, IBB);
2805 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
2806 "Invalid number of incoming values");
2810 case Instruction::ExtractElement: {
2811 if (canReuseExtract(E->Scalars, VL0)) {
2812 Value *V = VL0->getOperand(0);
2813 E->VectorizedValue = V;
2816 setInsertPointAfterBundle(E->Scalars, VL0);
2817 auto *V = Gather(E->Scalars, VecTy);
2818 E->VectorizedValue = V;
2821 case Instruction::ExtractValue: {
2822 if (canReuseExtract(E->Scalars, VL0)) {
2823 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2824 Builder.SetInsertPoint(LI);
2825 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2826 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2827 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2828 E->VectorizedValue = V;
2829 return propagateMetadata(V, E->Scalars);
2831 setInsertPointAfterBundle(E->Scalars, VL0);
2832 auto *V = Gather(E->Scalars, VecTy);
2833 E->VectorizedValue = V;
2836 case Instruction::ZExt:
2837 case Instruction::SExt:
2838 case Instruction::FPToUI:
2839 case Instruction::FPToSI:
2840 case Instruction::FPExt:
2841 case Instruction::PtrToInt:
2842 case Instruction::IntToPtr:
2843 case Instruction::SIToFP:
2844 case Instruction::UIToFP:
2845 case Instruction::Trunc:
2846 case Instruction::FPTrunc:
2847 case Instruction::BitCast: {
2849 for (Value *V : E->Scalars)
2850 INVL.push_back(cast<Instruction>(V)->getOperand(0));
2852 setInsertPointAfterBundle(E->Scalars, VL0);
2854 Value *InVec = vectorizeTree(INVL);
2856 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2859 CastInst *CI = dyn_cast<CastInst>(VL0);
2860 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2861 E->VectorizedValue = V;
2862 ++NumVectorInstructions;
2865 case Instruction::FCmp:
2866 case Instruction::ICmp: {
2867 ValueList LHSV, RHSV;
2868 for (Value *V : E->Scalars) {
2869 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2870 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2873 setInsertPointAfterBundle(E->Scalars, VL0);
2875 Value *L = vectorizeTree(LHSV);
2876 Value *R = vectorizeTree(RHSV);
2878 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2881 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2883 if (S.Opcode == Instruction::FCmp)
2884 V = Builder.CreateFCmp(P0, L, R);
2886 V = Builder.CreateICmp(P0, L, R);
2888 E->VectorizedValue = V;
2889 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2890 ++NumVectorInstructions;
2893 case Instruction::Select: {
2894 ValueList TrueVec, FalseVec, CondVec;
2895 for (Value *V : E->Scalars) {
2896 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2897 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2898 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2901 setInsertPointAfterBundle(E->Scalars, VL0);
2903 Value *Cond = vectorizeTree(CondVec);
2904 Value *True = vectorizeTree(TrueVec);
2905 Value *False = vectorizeTree(FalseVec);
2907 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2910 Value *V = Builder.CreateSelect(Cond, True, False);
2911 E->VectorizedValue = V;
2912 ++NumVectorInstructions;
2915 case Instruction::Add:
2916 case Instruction::FAdd:
2917 case Instruction::Sub:
2918 case Instruction::FSub:
2919 case Instruction::Mul:
2920 case Instruction::FMul:
2921 case Instruction::UDiv:
2922 case Instruction::SDiv:
2923 case Instruction::FDiv:
2924 case Instruction::URem:
2925 case Instruction::SRem:
2926 case Instruction::FRem:
2927 case Instruction::Shl:
2928 case Instruction::LShr:
2929 case Instruction::AShr:
2930 case Instruction::And:
2931 case Instruction::Or:
2932 case Instruction::Xor: {
2933 ValueList LHSVL, RHSVL;
2934 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2935 reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
2938 for (Value *V : E->Scalars) {
2939 auto *I = cast<Instruction>(V);
2940 LHSVL.push_back(I->getOperand(0));
2941 RHSVL.push_back(I->getOperand(1));
2944 setInsertPointAfterBundle(E->Scalars, VL0);
2946 Value *LHS = vectorizeTree(LHSVL);
2947 Value *RHS = vectorizeTree(RHSVL);
2949 if (Value *V = alreadyVectorized(E->Scalars, VL0))
2952 Value *V = Builder.CreateBinOp(
2953 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
2954 E->VectorizedValue = V;
2955 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2956 ++NumVectorInstructions;
2958 if (Instruction *I = dyn_cast<Instruction>(V))
2959 return propagateMetadata(I, E->Scalars);
2963 case Instruction::Load: {
2964 // Loads are inserted at the head of the tree because we don't want to
2965 // sink them all the way down past store instructions.
2966 setInsertPointAfterBundle(E->Scalars, VL0);
2968 LoadInst *LI = cast<LoadInst>(VL0);
2969 Type *ScalarLoadTy = LI->getType();
2970 unsigned AS = LI->getPointerAddressSpace();
2972 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2973 VecTy->getPointerTo(AS));
2975 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2976 // ExternalUses list to make sure that an extract will be generated in the
2978 Value *PO = LI->getPointerOperand();
2979 if (getTreeEntry(PO))
2980 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
2982 unsigned Alignment = LI->getAlignment();
2983 LI = Builder.CreateLoad(VecPtr);
2985 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2987 LI->setAlignment(Alignment);
2988 E->VectorizedValue = LI;
2989 ++NumVectorInstructions;
2990 return propagateMetadata(LI, E->Scalars);
2992 case Instruction::Store: {
2993 StoreInst *SI = cast<StoreInst>(VL0);
2994 unsigned Alignment = SI->getAlignment();
2995 unsigned AS = SI->getPointerAddressSpace();
2997 ValueList ScalarStoreValues;
2998 for (Value *V : E->Scalars)
2999 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3001 setInsertPointAfterBundle(E->Scalars, VL0);
3003 Value *VecValue = vectorizeTree(ScalarStoreValues);
3004 Value *ScalarPtr = SI->getPointerOperand();
3005 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3006 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3008 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3009 // ExternalUses to make sure that an extract will be generated in the
3011 if (getTreeEntry(ScalarPtr))
3012 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3015 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3017 S->setAlignment(Alignment);
3018 E->VectorizedValue = S;
3019 ++NumVectorInstructions;
3020 return propagateMetadata(S, E->Scalars);
3022 case Instruction::GetElementPtr: {
3023 setInsertPointAfterBundle(E->Scalars, VL0);
3026 for (Value *V : E->Scalars)
3027 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3029 Value *Op0 = vectorizeTree(Op0VL);
3031 std::vector<Value *> OpVecs;
3032 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3035 for (Value *V : E->Scalars)
3036 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3038 Value *OpVec = vectorizeTree(OpVL);
3039 OpVecs.push_back(OpVec);
3042 Value *V = Builder.CreateGEP(
3043 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3044 E->VectorizedValue = V;
3045 ++NumVectorInstructions;
3047 if (Instruction *I = dyn_cast<Instruction>(V))
3048 return propagateMetadata(I, E->Scalars);
3052 case Instruction::Call: {
3053 CallInst *CI = cast<CallInst>(VL0);
3054 setInsertPointAfterBundle(E->Scalars, VL0);
3056 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3057 Value *ScalarArg = nullptr;
3058 if (CI && (FI = CI->getCalledFunction())) {
3059 IID = FI->getIntrinsicID();
3061 std::vector<Value *> OpVecs;
3062 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3064 // ctlz,cttz and powi are special intrinsics whose second argument is
3065 // a scalar. This argument should not be vectorized.
3066 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3067 CallInst *CEI = cast<CallInst>(VL0);
3068 ScalarArg = CEI->getArgOperand(j);
3069 OpVecs.push_back(CEI->getArgOperand(j));
3072 for (Value *V : E->Scalars) {
3073 CallInst *CEI = cast<CallInst>(V);
3074 OpVL.push_back(CEI->getArgOperand(j));
3077 Value *OpVec = vectorizeTree(OpVL);
3078 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3079 OpVecs.push_back(OpVec);
3082 Module *M = F->getParent();
3083 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3084 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3085 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3086 SmallVector<OperandBundleDef, 1> OpBundles;
3087 CI->getOperandBundlesAsDefs(OpBundles);
3088 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3090 // The scalar argument uses an in-tree scalar so we add the new vectorized
3091 // call to ExternalUses list to make sure that an extract will be
3092 // generated in the future.
3093 if (ScalarArg && getTreeEntry(ScalarArg))
3094 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3096 E->VectorizedValue = V;
3097 propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3098 ++NumVectorInstructions;
3101 case Instruction::ShuffleVector: {
3102 ValueList LHSVL, RHSVL;
3103 assert(Instruction::isBinaryOp(S.Opcode) &&
3104 "Invalid Shuffle Vector Operand");
3105 reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3106 setInsertPointAfterBundle(E->Scalars, VL0);
3108 Value *LHS = vectorizeTree(LHSVL);
3109 Value *RHS = vectorizeTree(RHSVL);
3111 if (Value *V = alreadyVectorized(E->Scalars, VL0))
3114 // Create a vector of LHS op1 RHS
3115 Value *V0 = Builder.CreateBinOp(
3116 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3118 unsigned AltOpcode = getAltOpcode(S.Opcode);
3119 // Create a vector of LHS op2 RHS
3120 Value *V1 = Builder.CreateBinOp(
3121 static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3123 // Create shuffle to take alternate operations from the vector.
3124 // Also, gather up odd and even scalar ops to propagate IR flags to
3125 // each vector operation.
3126 ValueList OddScalars, EvenScalars;
3127 unsigned e = E->Scalars.size();
3128 SmallVector<Constant *, 8> Mask(e);
3129 for (unsigned i = 0; i < e; ++i) {
3131 Mask[i] = Builder.getInt32(e + i);
3132 OddScalars.push_back(E->Scalars[i]);
3134 Mask[i] = Builder.getInt32(i);
3135 EvenScalars.push_back(E->Scalars[i]);
3139 Value *ShuffleMask = ConstantVector::get(Mask);
3140 propagateIRFlags(V0, EvenScalars);
3141 propagateIRFlags(V1, OddScalars);
3143 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3144 E->VectorizedValue = V;
3145 ++NumVectorInstructions;
3146 if (Instruction *I = dyn_cast<Instruction>(V))
3147 return propagateMetadata(I, E->Scalars);
3152 llvm_unreachable("unknown inst");
3157 Value *BoUpSLP::vectorizeTree() {
3158 ExtraValueToDebugLocsMap ExternallyUsedValues;
3159 return vectorizeTree(ExternallyUsedValues);
3163 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3164 // All blocks must be scheduled before any instructions are inserted.
3165 for (auto &BSIter : BlocksSchedules) {
3166 scheduleBlock(BSIter.second.get());
3169 Builder.SetInsertPoint(&F->getEntryBlock().front());
3170 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3172 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3173 // vectorized root. InstCombine will then rewrite the entire expression. We
3174 // sign extend the extracted values below.
3175 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3176 if (MinBWs.count(ScalarRoot)) {
3177 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3178 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3179 auto BundleWidth = VectorizableTree[0].Scalars.size();
3180 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3181 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3182 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3183 VectorizableTree[0].VectorizedValue = Trunc;
3186 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n");
3188 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3189 // specified by ScalarType.
3190 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3191 if (!MinBWs.count(ScalarRoot))
3193 if (MinBWs[ScalarRoot].second)
3194 return Builder.CreateSExt(Ex, ScalarType);
3195 return Builder.CreateZExt(Ex, ScalarType);
3198 // Extract all of the elements with the external uses.
3199 for (const auto &ExternalUse : ExternalUses) {
3200 Value *Scalar = ExternalUse.Scalar;
3201 llvm::User *User = ExternalUse.User;
3203 // Skip users that we already RAUW. This happens when one instruction
3204 // has multiple uses of the same value.
3205 if (User && !is_contained(Scalar->users(), User))
3207 TreeEntry *E = getTreeEntry(Scalar);
3208 assert(E && "Invalid scalar");
3209 assert(!E->NeedToGather && "Extracting from a gather list");
3211 Value *Vec = E->VectorizedValue;
3212 assert(Vec && "Can't find vectorizable value");
3214 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3215 // If User == nullptr, the Scalar is used as extra arg. Generate
3216 // ExtractElement instruction and update the record for this scalar in
3217 // ExternallyUsedValues.
3219 assert(ExternallyUsedValues.count(Scalar) &&
3220 "Scalar with nullptr as an external user must be registered in "
3221 "ExternallyUsedValues map");
3222 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3223 Builder.SetInsertPoint(VecI->getParent(),
3224 std::next(VecI->getIterator()));
3226 Builder.SetInsertPoint(&F->getEntryBlock().front());
3228 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3229 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3230 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3231 auto &Locs = ExternallyUsedValues[Scalar];
3232 ExternallyUsedValues.insert({Ex, Locs});
3233 ExternallyUsedValues.erase(Scalar);
3237 // Generate extracts for out-of-tree users.
3238 // Find the insertion point for the extractelement lane.
3239 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3240 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3241 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3242 if (PH->getIncomingValue(i) == Scalar) {
3243 TerminatorInst *IncomingTerminator =
3244 PH->getIncomingBlock(i)->getTerminator();
3245 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3246 Builder.SetInsertPoint(VecI->getParent(),
3247 std::next(VecI->getIterator()));
3249 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3251 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3252 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3253 CSEBlocks.insert(PH->getIncomingBlock(i));
3254 PH->setOperand(i, Ex);
3258 Builder.SetInsertPoint(cast<Instruction>(User));
3259 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3260 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3261 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3262 User->replaceUsesOfWith(Scalar, Ex);
3265 Builder.SetInsertPoint(&F->getEntryBlock().front());
3266 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3267 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3268 CSEBlocks.insert(&F->getEntryBlock());
3269 User->replaceUsesOfWith(Scalar, Ex);
3272 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3275 // For each vectorized value:
3276 for (TreeEntry &EIdx : VectorizableTree) {
3277 TreeEntry *Entry = &EIdx;
3279 // No need to handle users of gathered values.
3280 if (Entry->NeedToGather)
3283 assert(Entry->VectorizedValue && "Can't find vectorizable value");
3286 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3287 Value *Scalar = Entry->Scalars[Lane];
3289 Type *Ty = Scalar->getType();
3290 if (!Ty->isVoidTy()) {
3292 for (User *U : Scalar->users()) {
3293 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3295 // It is legal to replace users in the ignorelist by undef.
3296 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3297 "Replacing out-of-tree value with undef");
3300 Value *Undef = UndefValue::get(Ty);
3301 Scalar->replaceAllUsesWith(Undef);
3303 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3304 eraseInstruction(cast<Instruction>(Scalar));
3308 Builder.ClearInsertionPoint();
3310 return VectorizableTree[0].VectorizedValue;
3313 void BoUpSLP::optimizeGatherSequence(Function &F) {
3314 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3315 << " gather sequences instructions.\n");
3316 // LICM InsertElementInst sequences.
3317 for (Instruction *it : GatherSeq) {
3318 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
3323 // Check if this block is inside a loop.
3324 Loop *L = LI->getLoopFor(Insert->getParent());
3328 // Check if it has a preheader.
3329 BasicBlock *PreHeader = L->getLoopPreheader();
3333 // If the vector or the element that we insert into it are
3334 // instructions that are defined in this basic block then we can't
3335 // hoist this instruction.
3336 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
3337 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
3338 if (CurrVec && L->contains(CurrVec))
3340 if (NewElem && L->contains(NewElem))
3343 // We can hoist this instruction. Move it to the pre-header.
3344 Insert->moveBefore(PreHeader->getTerminator());
3347 // Perform O(N^2) search over the gather sequences and merge identical
3348 // instructions. TODO: We can further optimize this scan if we split the
3349 // instructions into different buckets based on the insert lane.
3350 SmallVector<Instruction *, 16> Visited;
3351 ReversePostOrderTraversal<Function *> RPOT(&F);
3352 for (auto BB : RPOT) {
3353 // Traverse CSEBlocks by RPOT order.
3354 if (!CSEBlocks.count(BB))
3357 // For all instructions in blocks containing gather sequences:
3358 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3359 Instruction *In = &*it++;
3360 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3363 // Check if we can replace this instruction with any of the
3364 // visited instructions.
3365 for (Instruction *v : Visited) {
3366 if (In->isIdenticalTo(v) &&
3367 DT->dominates(v->getParent(), In->getParent())) {
3368 In->replaceAllUsesWith(v);
3369 eraseInstruction(In);
3375 assert(!is_contained(Visited, In));
3376 Visited.push_back(In);
3384 // Groups the instructions to a bundle (which is then a single scheduling entity)
3385 // and schedules instructions until the bundle gets ready.
3386 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3387 BoUpSLP *SLP, Value *OpValue) {
3388 if (isa<PHINode>(OpValue))
3391 // Initialize the instruction bundle.
3392 Instruction *OldScheduleEnd = ScheduleEnd;
3393 ScheduleData *PrevInBundle = nullptr;
3394 ScheduleData *Bundle = nullptr;
3395 bool ReSchedule = false;
3396 DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n");
3398 // Make sure that the scheduling region contains all
3399 // instructions of the bundle.
3400 for (Value *V : VL) {
3401 if (!extendSchedulingRegion(V, OpValue))
3405 for (Value *V : VL) {
3406 ScheduleData *BundleMember = getScheduleData(V);
3407 assert(BundleMember &&
3408 "no ScheduleData for bundle member (maybe not in same basic block)");
3409 if (BundleMember->IsScheduled) {
3410 // A bundle member was scheduled as single instruction before and now
3411 // needs to be scheduled as part of the bundle. We just get rid of the
3412 // existing schedule.
3413 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3414 << " was already scheduled\n");
3417 assert(BundleMember->isSchedulingEntity() &&
3418 "bundle member already part of other bundle");
3420 PrevInBundle->NextInBundle = BundleMember;
3422 Bundle = BundleMember;
3424 BundleMember->UnscheduledDepsInBundle = 0;
3425 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3427 // Group the instructions to a bundle.
3428 BundleMember->FirstInBundle = Bundle;
3429 PrevInBundle = BundleMember;
3431 if (ScheduleEnd != OldScheduleEnd) {
3432 // The scheduling region got new instructions at the lower end (or it is a
3433 // new region for the first bundle). This makes it necessary to
3434 // recalculate all dependencies.
3435 // It is seldom that this needs to be done a second time after adding the
3436 // initial bundle to the region.
3437 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3438 doForAllOpcodes(I, [](ScheduleData *SD) {
3439 SD->clearDependencies();
3446 initialFillReadyList(ReadyInsts);
3449 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3450 << BB->getName() << "\n");
3452 calculateDependencies(Bundle, true, SLP);
3454 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3455 // means that there are no cyclic dependencies and we can schedule it.
3456 // Note that's important that we don't "schedule" the bundle yet (see
3457 // cancelScheduling).
3458 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3460 ScheduleData *pickedSD = ReadyInsts.back();
3461 ReadyInsts.pop_back();
3463 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3464 schedule(pickedSD, ReadyInsts);
3467 if (!Bundle->isReady()) {
3468 cancelScheduling(VL, OpValue);
3474 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3476 if (isa<PHINode>(OpValue))
3479 ScheduleData *Bundle = getScheduleData(OpValue);
3480 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3481 assert(!Bundle->IsScheduled &&
3482 "Can't cancel bundle which is already scheduled");
3483 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3484 "tried to unbundle something which is not a bundle");
3486 // Un-bundle: make single instructions out of the bundle.
3487 ScheduleData *BundleMember = Bundle;
3488 while (BundleMember) {
3489 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3490 BundleMember->FirstInBundle = BundleMember;
3491 ScheduleData *Next = BundleMember->NextInBundle;
3492 BundleMember->NextInBundle = nullptr;
3493 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3494 if (BundleMember->UnscheduledDepsInBundle == 0) {
3495 ReadyInsts.insert(BundleMember);
3497 BundleMember = Next;
3501 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3502 // Allocate a new ScheduleData for the instruction.
3503 if (ChunkPos >= ChunkSize) {
3504 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3507 return &(ScheduleDataChunks.back()[ChunkPos++]);
3510 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3512 if (getScheduleData(V, isOneOf(OpValue, V)))
3514 Instruction *I = dyn_cast<Instruction>(V);
3515 assert(I && "bundle member must be an instruction");
3516 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3517 auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3518 ScheduleData *ISD = getScheduleData(I);
3521 assert(isInSchedulingRegion(ISD) &&
3522 "ScheduleData not in scheduling region");
3523 ScheduleData *SD = allocateScheduleDataChunks();
3525 SD->init(SchedulingRegionID, OpValue);
3526 ExtraScheduleDataMap[I][OpValue] = SD;
3529 if (CheckSheduleForI(I))
3531 if (!ScheduleStart) {
3532 // It's the first instruction in the new region.
3533 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3535 ScheduleEnd = I->getNextNode();
3536 if (isOneOf(OpValue, I) != I)
3537 CheckSheduleForI(I);
3538 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3539 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3542 // Search up and down at the same time, because we don't know if the new
3543 // instruction is above or below the existing scheduling region.
3544 BasicBlock::reverse_iterator UpIter =
3545 ++ScheduleStart->getIterator().getReverse();
3546 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3547 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3548 BasicBlock::iterator LowerEnd = BB->end();
3550 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3551 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3555 if (UpIter != UpperEnd) {
3556 if (&*UpIter == I) {
3557 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3559 if (isOneOf(OpValue, I) != I)
3560 CheckSheduleForI(I);
3561 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n");
3566 if (DownIter != LowerEnd) {
3567 if (&*DownIter == I) {
3568 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3570 ScheduleEnd = I->getNextNode();
3571 if (isOneOf(OpValue, I) != I)
3572 CheckSheduleForI(I);
3573 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3574 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
3579 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
3580 "instruction not found in block");
3585 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3587 ScheduleData *PrevLoadStore,
3588 ScheduleData *NextLoadStore) {
3589 ScheduleData *CurrentLoadStore = PrevLoadStore;
3590 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3591 ScheduleData *SD = ScheduleDataMap[I];
3593 SD = allocateScheduleDataChunks();
3594 ScheduleDataMap[I] = SD;
3597 assert(!isInSchedulingRegion(SD) &&
3598 "new ScheduleData already in scheduling region");
3599 SD->init(SchedulingRegionID, I);
3601 if (I->mayReadOrWriteMemory() &&
3602 (!isa<IntrinsicInst>(I) ||
3603 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
3604 // Update the linked list of memory accessing instructions.
3605 if (CurrentLoadStore) {
3606 CurrentLoadStore->NextLoadStore = SD;
3608 FirstLoadStoreInRegion = SD;
3610 CurrentLoadStore = SD;
3613 if (NextLoadStore) {
3614 if (CurrentLoadStore)
3615 CurrentLoadStore->NextLoadStore = NextLoadStore;
3617 LastLoadStoreInRegion = CurrentLoadStore;
3621 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3622 bool InsertInReadyList,
3624 assert(SD->isSchedulingEntity());
3626 SmallVector<ScheduleData *, 10> WorkList;
3627 WorkList.push_back(SD);
3629 while (!WorkList.empty()) {
3630 ScheduleData *SD = WorkList.back();
3631 WorkList.pop_back();
3633 ScheduleData *BundleMember = SD;
3634 while (BundleMember) {
3635 assert(isInSchedulingRegion(BundleMember));
3636 if (!BundleMember->hasValidDependencies()) {
3638 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n");
3639 BundleMember->Dependencies = 0;
3640 BundleMember->resetUnscheduledDeps();
3642 // Handle def-use chain dependencies.
3643 if (BundleMember->OpValue != BundleMember->Inst) {
3644 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
3645 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3646 BundleMember->Dependencies++;
3647 ScheduleData *DestBundle = UseSD->FirstInBundle;
3648 if (!DestBundle->IsScheduled)
3649 BundleMember->incrementUnscheduledDeps(1);
3650 if (!DestBundle->hasValidDependencies())
3651 WorkList.push_back(DestBundle);
3654 for (User *U : BundleMember->Inst->users()) {
3655 if (isa<Instruction>(U)) {
3656 ScheduleData *UseSD = getScheduleData(U);
3657 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3658 BundleMember->Dependencies++;
3659 ScheduleData *DestBundle = UseSD->FirstInBundle;
3660 if (!DestBundle->IsScheduled)
3661 BundleMember->incrementUnscheduledDeps(1);
3662 if (!DestBundle->hasValidDependencies())
3663 WorkList.push_back(DestBundle);
3666 // I'm not sure if this can ever happen. But we need to be safe.
3667 // This lets the instruction/bundle never be scheduled and
3668 // eventually disable vectorization.
3669 BundleMember->Dependencies++;
3670 BundleMember->incrementUnscheduledDeps(1);
3675 // Handle the memory dependencies.
3676 ScheduleData *DepDest = BundleMember->NextLoadStore;
3678 Instruction *SrcInst = BundleMember->Inst;
3679 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3680 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3681 unsigned numAliased = 0;
3682 unsigned DistToSrc = 1;
3685 assert(isInSchedulingRegion(DepDest));
3687 // We have two limits to reduce the complexity:
3688 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3689 // SLP->isAliased (which is the expensive part in this loop).
3690 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3691 // the whole loop (even if the loop is fast, it's quadratic).
3692 // It's important for the loop break condition (see below) to
3693 // check this limit even between two read-only instructions.
3694 if (DistToSrc >= MaxMemDepDistance ||
3695 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3696 (numAliased >= AliasedCheckLimit ||
3697 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3699 // We increment the counter only if the locations are aliased
3700 // (instead of counting all alias checks). This gives a better
3701 // balance between reduced runtime and accurate dependencies.
3704 DepDest->MemoryDependencies.push_back(BundleMember);
3705 BundleMember->Dependencies++;
3706 ScheduleData *DestBundle = DepDest->FirstInBundle;
3707 if (!DestBundle->IsScheduled) {
3708 BundleMember->incrementUnscheduledDeps(1);
3710 if (!DestBundle->hasValidDependencies()) {
3711 WorkList.push_back(DestBundle);
3714 DepDest = DepDest->NextLoadStore;
3716 // Example, explaining the loop break condition: Let's assume our
3717 // starting instruction is i0 and MaxMemDepDistance = 3.
3720 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3723 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3724 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3725 // Previously we already added dependencies from i3 to i6,i7,i8
3726 // (because of MaxMemDepDistance). As we added a dependency from
3727 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3728 // and we can abort this loop at i6.
3729 if (DistToSrc >= 2 * MaxMemDepDistance)
3735 BundleMember = BundleMember->NextInBundle;
3737 if (InsertInReadyList && SD->isReady()) {
3738 ReadyInsts.push_back(SD);
3739 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n");
3744 void BoUpSLP::BlockScheduling::resetSchedule() {
3745 assert(ScheduleStart &&
3746 "tried to reset schedule on block which has not been scheduled");
3747 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3748 doForAllOpcodes(I, [&](ScheduleData *SD) {
3749 assert(isInSchedulingRegion(SD) &&
3750 "ScheduleData not in scheduling region");
3751 SD->IsScheduled = false;
3752 SD->resetUnscheduledDeps();
3758 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3759 if (!BS->ScheduleStart)
3762 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
3764 BS->resetSchedule();
3766 // For the real scheduling we use a more sophisticated ready-list: it is
3767 // sorted by the original instruction location. This lets the final schedule
3768 // be as close as possible to the original instruction order.
3769 struct ScheduleDataCompare {
3770 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3771 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3774 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3776 // Ensure that all dependency data is updated and fill the ready-list with
3777 // initial instructions.
3779 int NumToSchedule = 0;
3780 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3781 I = I->getNextNode()) {
3782 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
3783 assert(SD->isPartOfBundle() ==
3784 (getTreeEntry(SD->Inst) != nullptr) &&
3785 "scheduler and vectorizer bundle mismatch");
3786 SD->FirstInBundle->SchedulingPriority = Idx++;
3787 if (SD->isSchedulingEntity()) {
3788 BS->calculateDependencies(SD, false, this);
3793 BS->initialFillReadyList(ReadyInsts);
3795 Instruction *LastScheduledInst = BS->ScheduleEnd;
3797 // Do the "real" scheduling.
3798 while (!ReadyInsts.empty()) {
3799 ScheduleData *picked = *ReadyInsts.begin();
3800 ReadyInsts.erase(ReadyInsts.begin());
3802 // Move the scheduled instruction(s) to their dedicated places, if not
3804 ScheduleData *BundleMember = picked;
3805 while (BundleMember) {
3806 Instruction *pickedInst = BundleMember->Inst;
3807 if (LastScheduledInst->getNextNode() != pickedInst) {
3808 BS->BB->getInstList().remove(pickedInst);
3809 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3812 LastScheduledInst = pickedInst;
3813 BundleMember = BundleMember->NextInBundle;
3816 BS->schedule(picked, ReadyInsts);
3819 assert(NumToSchedule == 0 && "could not schedule all instructions");
3821 // Avoid duplicate scheduling of the block.
3822 BS->ScheduleStart = nullptr;
3825 unsigned BoUpSLP::getVectorElementSize(Value *V) {
3826 // If V is a store, just return the width of the stored value without
3827 // traversing the expression tree. This is the common case.
3828 if (auto *Store = dyn_cast<StoreInst>(V))
3829 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3831 // If V is not a store, we can traverse the expression tree to find loads
3832 // that feed it. The type of the loaded value may indicate a more suitable
3833 // width than V's type. We want to base the vector element size on the width
3834 // of memory operations where possible.
3835 SmallVector<Instruction *, 16> Worklist;
3836 SmallPtrSet<Instruction *, 16> Visited;
3837 if (auto *I = dyn_cast<Instruction>(V))
3838 Worklist.push_back(I);
3840 // Traverse the expression tree in bottom-up order looking for loads. If we
3841 // encounter an instruciton we don't yet handle, we give up.
3843 auto FoundUnknownInst = false;
3844 while (!Worklist.empty() && !FoundUnknownInst) {
3845 auto *I = Worklist.pop_back_val();
3848 // We should only be looking at scalar instructions here. If the current
3849 // instruction has a vector type, give up.
3850 auto *Ty = I->getType();
3851 if (isa<VectorType>(Ty))
3852 FoundUnknownInst = true;
3854 // If the current instruction is a load, update MaxWidth to reflect the
3855 // width of the loaded value.
3856 else if (isa<LoadInst>(I))
3857 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3859 // Otherwise, we need to visit the operands of the instruction. We only
3860 // handle the interesting cases from buildTree here. If an operand is an
3861 // instruction we haven't yet visited, we add it to the worklist.
3862 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3863 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3864 for (Use &U : I->operands())
3865 if (auto *J = dyn_cast<Instruction>(U.get()))
3866 if (!Visited.count(J))
3867 Worklist.push_back(J);
3870 // If we don't yet handle the instruction, give up.
3872 FoundUnknownInst = true;
3875 // If we didn't encounter a memory access in the expression tree, or if we
3876 // gave up for some reason, just return the width of V.
3877 if (!MaxWidth || FoundUnknownInst)
3878 return DL->getTypeSizeInBits(V->getType());
3880 // Otherwise, return the maximum width we found.
3884 // Determine if a value V in a vectorizable expression Expr can be demoted to a
3885 // smaller type with a truncation. We collect the values that will be demoted
3886 // in ToDemote and additional roots that require investigating in Roots.
3887 static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
3888 SmallVectorImpl<Value *> &ToDemote,
3889 SmallVectorImpl<Value *> &Roots) {
3890 // We can always demote constants.
3891 if (isa<Constant>(V)) {
3892 ToDemote.push_back(V);
3896 // If the value is not an instruction in the expression with only one use, it
3897 // cannot be demoted.
3898 auto *I = dyn_cast<Instruction>(V);
3899 if (!I || !I->hasOneUse() || !Expr.count(I))
3902 switch (I->getOpcode()) {
3904 // We can always demote truncations and extensions. Since truncations can
3905 // seed additional demotion, we save the truncated value.
3906 case Instruction::Trunc:
3907 Roots.push_back(I->getOperand(0));
3909 case Instruction::ZExt:
3910 case Instruction::SExt:
3913 // We can demote certain binary operations if we can demote both of their
3915 case Instruction::Add:
3916 case Instruction::Sub:
3917 case Instruction::Mul:
3918 case Instruction::And:
3919 case Instruction::Or:
3920 case Instruction::Xor:
3921 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3922 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3926 // We can demote selects if we can demote their true and false values.
3927 case Instruction::Select: {
3928 SelectInst *SI = cast<SelectInst>(I);
3929 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3930 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3935 // We can demote phis if we can demote all their incoming operands. Note that
3936 // we don't need to worry about cycles since we ensure single use above.
3937 case Instruction::PHI: {
3938 PHINode *PN = cast<PHINode>(I);
3939 for (Value *IncValue : PN->incoming_values())
3940 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3945 // Otherwise, conservatively give up.
3950 // Record the value that we can demote.
3951 ToDemote.push_back(V);
3955 void BoUpSLP::computeMinimumValueSizes() {
3956 // If there are no external uses, the expression tree must be rooted by a
3957 // store. We can't demote in-memory values, so there is nothing to do here.
3958 if (ExternalUses.empty())
3961 // We only attempt to truncate integer expressions.
3962 auto &TreeRoot = VectorizableTree[0].Scalars;
3963 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3967 // If the expression is not rooted by a store, these roots should have
3968 // external uses. We will rely on InstCombine to rewrite the expression in
3969 // the narrower type. However, InstCombine only rewrites single-use values.
3970 // This means that if a tree entry other than a root is used externally, it
3971 // must have multiple uses and InstCombine will not rewrite it. The code
3972 // below ensures that only the roots are used externally.
3973 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3974 for (auto &EU : ExternalUses)
3975 if (!Expr.erase(EU.Scalar))
3980 // Collect the scalar values of the vectorizable expression. We will use this
3981 // context to determine which values can be demoted. If we see a truncation,
3982 // we mark it as seeding another demotion.
3983 for (auto &Entry : VectorizableTree)
3984 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3986 // Ensure the roots of the vectorizable tree don't form a cycle. They must
3987 // have a single external user that is not in the vectorizable tree.
3988 for (auto *Root : TreeRoot)
3989 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
3992 // Conservatively determine if we can actually truncate the roots of the
3993 // expression. Collect the values that can be demoted in ToDemote and
3994 // additional roots that require investigating in Roots.
3995 SmallVector<Value *, 32> ToDemote;
3996 SmallVector<Value *, 4> Roots;
3997 for (auto *Root : TreeRoot)
3998 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4001 // The maximum bit width required to represent all the values that can be
4002 // demoted without loss of precision. It would be safe to truncate the roots
4003 // of the expression to this width.
4004 auto MaxBitWidth = 8u;
4006 // We first check if all the bits of the roots are demanded. If they're not,
4007 // we can truncate the roots to this narrower type.
4008 for (auto *Root : TreeRoot) {
4009 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4010 MaxBitWidth = std::max<unsigned>(
4011 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4014 // True if the roots can be zero-extended back to their original type, rather
4015 // than sign-extended. We know that if the leading bits are not demanded, we
4016 // can safely zero-extend. So we initialize IsKnownPositive to True.
4017 bool IsKnownPositive = true;
4019 // If all the bits of the roots are demanded, we can try a little harder to
4020 // compute a narrower type. This can happen, for example, if the roots are
4021 // getelementptr indices. InstCombine promotes these indices to the pointer
4022 // width. Thus, all their bits are technically demanded even though the
4023 // address computation might be vectorized in a smaller type.
4025 // We start by looking at each entry that can be demoted. We compute the
4026 // maximum bit width required to store the scalar by using ValueTracking to
4027 // compute the number of high-order bits we can truncate.
4028 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
4031 // Determine if the sign bit of all the roots is known to be zero. If not,
4032 // IsKnownPositive is set to False.
4033 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4034 KnownBits Known = computeKnownBits(R, *DL);
4035 return Known.isNonNegative();
4038 // Determine the maximum number of bits required to store the scalar
4040 for (auto *Scalar : ToDemote) {
4041 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4042 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4043 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4046 // If we can't prove that the sign bit is zero, we must add one to the
4047 // maximum bit width to account for the unknown sign bit. This preserves
4048 // the existing sign bit so we can safely sign-extend the root back to the
4049 // original type. Otherwise, if we know the sign bit is zero, we will
4050 // zero-extend the root instead.
4052 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4053 // one to the maximum bit width will yield a larger-than-necessary
4054 // type. In general, we need to add an extra bit only if we can't
4055 // prove that the upper bit of the original type is equal to the
4056 // upper bit of the proposed smaller type. If these two bits are the
4057 // same (either zero or one) we know that sign-extending from the
4058 // smaller type will result in the same value. Here, since we can't
4059 // yet prove this, we are just making the proposed smaller type
4060 // larger to ensure correctness.
4061 if (!IsKnownPositive)
4065 // Round MaxBitWidth up to the next power-of-two.
4066 if (!isPowerOf2_64(MaxBitWidth))
4067 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4069 // If the maximum bit width we compute is less than the with of the roots'
4070 // type, we can proceed with the narrowing. Otherwise, do nothing.
4071 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4074 // If we can truncate the root, we must collect additional values that might
4075 // be demoted as a result. That is, those seeded by truncations we will
4077 while (!Roots.empty())
4078 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4080 // Finally, map the values we can demote to the maximum bit with we computed.
4081 for (auto *Scalar : ToDemote)
4082 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4087 /// The SLPVectorizer Pass.
4088 struct SLPVectorizer : public FunctionPass {
4089 SLPVectorizerPass Impl;
4091 /// Pass identification, replacement for typeid
4094 explicit SLPVectorizer() : FunctionPass(ID) {
4095 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4098 bool doInitialization(Module &M) override {
4102 bool runOnFunction(Function &F) override {
4103 if (skipFunction(F))
4106 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4107 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4108 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4109 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4110 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4111 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4112 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4113 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4114 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4115 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4117 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4120 void getAnalysisUsage(AnalysisUsage &AU) const override {
4121 FunctionPass::getAnalysisUsage(AU);
4122 AU.addRequired<AssumptionCacheTracker>();
4123 AU.addRequired<ScalarEvolutionWrapperPass>();
4124 AU.addRequired<AAResultsWrapperPass>();
4125 AU.addRequired<TargetTransformInfoWrapperPass>();
4126 AU.addRequired<LoopInfoWrapperPass>();
4127 AU.addRequired<DominatorTreeWrapperPass>();
4128 AU.addRequired<DemandedBitsWrapperPass>();
4129 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4130 AU.addPreserved<LoopInfoWrapperPass>();
4131 AU.addPreserved<DominatorTreeWrapperPass>();
4132 AU.addPreserved<AAResultsWrapperPass>();
4133 AU.addPreserved<GlobalsAAWrapperPass>();
4134 AU.setPreservesCFG();
4138 } // end anonymous namespace
4140 PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4141 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4142 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4143 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4144 auto *AA = &AM.getResult<AAManager>(F);
4145 auto *LI = &AM.getResult<LoopAnalysis>(F);
4146 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4147 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4148 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4149 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4151 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4153 return PreservedAnalyses::all();
4155 PreservedAnalyses PA;
4156 PA.preserveSet<CFGAnalyses>();
4157 PA.preserve<AAManager>();
4158 PA.preserve<GlobalsAA>();
4162 bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4163 TargetTransformInfo *TTI_,
4164 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4165 LoopInfo *LI_, DominatorTree *DT_,
4166 AssumptionCache *AC_, DemandedBits *DB_,
4167 OptimizationRemarkEmitter *ORE_) {
4176 DL = &F.getParent()->getDataLayout();
4180 bool Changed = false;
4182 // If the target claims to have no vector registers don't attempt
4184 if (!TTI->getNumberOfRegisters(true))
4187 // Don't vectorize when the attribute NoImplicitFloat is used.
4188 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4191 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4193 // Use the bottom up slp vectorizer to construct chains that start with
4194 // store instructions.
4195 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4197 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4198 // delete instructions.
4200 // Scan the blocks in the function in post order.
4201 for (auto BB : post_order(&F.getEntryBlock())) {
4202 collectSeedInstructions(BB);
4204 // Vectorize trees that end at stores.
4205 if (!Stores.empty()) {
4206 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4207 << " underlying objects.\n");
4208 Changed |= vectorizeStoreChains(R);
4211 // Vectorize trees that end at reductions.
4212 Changed |= vectorizeChainsInBlock(BB, R);
4214 // Vectorize the index computations of getelementptr instructions. This
4215 // is primarily intended to catch gather-like idioms ending at
4216 // non-consecutive loads.
4217 if (!GEPs.empty()) {
4218 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4219 << " underlying objects.\n");
4220 Changed |= vectorizeGEPIndices(BB, R);
4225 R.optimizeGatherSequence(F);
4226 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4227 DEBUG(verifyFunction(F));
4232 /// \brief Check that the Values in the slice in VL array are still existent in
4233 /// the WeakTrackingVH array.
4234 /// Vectorization of part of the VL array may cause later values in the VL array
4235 /// to become invalid. We track when this has happened in the WeakTrackingVH
4237 static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4238 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4239 unsigned SliceSize) {
4240 VL = VL.slice(SliceBegin, SliceSize);
4241 VH = VH.slice(SliceBegin, SliceSize);
4242 return !std::equal(VL.begin(), VL.end(), VH.begin());
4245 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4246 unsigned VecRegSize) {
4247 unsigned ChainLen = Chain.size();
4248 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4250 unsigned Sz = R.getVectorElementSize(Chain[0]);
4251 unsigned VF = VecRegSize / Sz;
4253 if (!isPowerOf2_32(Sz) || VF < 2)
4256 // Keep track of values that were deleted by vectorizing in the loop below.
4257 SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4259 bool Changed = false;
4260 // Look for profitable vectorizable trees at all offsets, starting at zero.
4261 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
4265 // Check that a previous iteration of this loop did not delete the Value.
4266 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4269 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4271 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4273 R.buildTree(Operands);
4274 if (R.isTreeTinyAndNotFullyVectorizable())
4277 R.computeMinimumValueSizes();
4279 int Cost = R.getTreeCost();
4281 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
4282 if (Cost < -SLPCostThreshold) {
4283 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4285 using namespace ore;
4287 R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4288 cast<StoreInst>(Chain[i]))
4289 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4290 << " and with tree size "
4291 << NV("TreeSize", R.getTreeSize()));
4295 // Move to the next bundle.
4304 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4306 SetVector<StoreInst *> Heads;
4307 SmallDenseSet<StoreInst *> Tails;
4308 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4310 // We may run into multiple chains that merge into a single chain. We mark the
4311 // stores that we vectorized so that we don't visit the same store twice.
4312 BoUpSLP::ValueSet VectorizedStores;
4313 bool Changed = false;
4315 // Do a quadratic search on all of the given stores in reverse order and find
4316 // all of the pairs of stores that follow each other.
4317 SmallVector<unsigned, 16> IndexQueue;
4318 unsigned E = Stores.size();
4319 IndexQueue.resize(E - 1);
4320 for (unsigned I = E; I > 0; --I) {
4321 unsigned Idx = I - 1;
4322 // If a store has multiple consecutive store candidates, search Stores
4323 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4324 // This is because usually pairing with immediate succeeding or preceding
4325 // candidate create the best chance to find slp vectorization opportunity.
4326 unsigned Offset = 1;
4328 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4329 if (Idx >= Offset) {
4330 IndexQueue[Cnt] = Idx - Offset;
4333 if (Idx + Offset < E) {
4334 IndexQueue[Cnt] = Idx + Offset;
4339 for (auto K : IndexQueue) {
4340 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4341 Tails.insert(Stores[Idx]);
4342 Heads.insert(Stores[K]);
4343 ConsecutiveChain[Stores[K]] = Stores[Idx];
4349 // For stores that start but don't end a link in the chain:
4350 for (auto *SI : llvm::reverse(Heads)) {
4351 if (Tails.count(SI))
4354 // We found a store instr that starts a chain. Now follow the chain and try
4356 BoUpSLP::ValueList Operands;
4358 // Collect the chain into a list.
4359 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4360 Operands.push_back(I);
4361 // Move to the next value in the chain.
4362 I = ConsecutiveChain[I];
4365 // FIXME: Is division-by-2 the correct step? Should we assert that the
4366 // register size is a power-of-2?
4367 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4369 if (vectorizeStoreChain(Operands, R, Size)) {
4370 // Mark the vectorized stores so that we don't vectorize them again.
4371 VectorizedStores.insert(Operands.begin(), Operands.end());
4381 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4382 // Initialize the collections. We will make a single pass over the block.
4386 // Visit the store and getelementptr instructions in BB and organize them in
4387 // Stores and GEPs according to the underlying objects of their pointer
4389 for (Instruction &I : *BB) {
4390 // Ignore store instructions that are volatile or have a pointer operand
4391 // that doesn't point to a scalar type.
4392 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4393 if (!SI->isSimple())
4395 if (!isValidElementType(SI->getValueOperand()->getType()))
4397 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4400 // Ignore getelementptr instructions that have more than one index, a
4401 // constant index, or a pointer operand that doesn't point to a scalar
4403 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4404 auto Idx = GEP->idx_begin()->get();
4405 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4407 if (!isValidElementType(Idx->getType()))
4409 if (GEP->getType()->isVectorTy())
4411 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4416 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4419 Value *VL[] = { A, B };
4420 return tryToVectorizeList(VL, R, None, true);
4423 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4424 ArrayRef<Value *> BuildVector,
4426 bool NeedExtraction) {
4430 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()
4433 // Check that all of the parts are scalar instructions of the same type.
4434 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4438 unsigned Opcode0 = I0->getOpcode();
4440 unsigned Sz = R.getVectorElementSize(I0);
4441 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4442 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4444 R.getORE()->emit([&]() {
4445 return OptimizationRemarkMissed(
4446 SV_NAME, "SmallVF", I0)
4447 << "Cannot SLP vectorize list: vectorization factor "
4448 << "less than 2 is not supported";
4453 for (Value *V : VL) {
4454 Type *Ty = V->getType();
4455 if (!isValidElementType(Ty)) {
4456 // NOTE: the following will give user internal llvm type name, which may not be useful
4457 R.getORE()->emit([&]() {
4458 std::string type_str;
4459 llvm::raw_string_ostream rso(type_str);
4461 return OptimizationRemarkMissed(
4462 SV_NAME, "UnsupportedType", I0)
4463 << "Cannot SLP vectorize list: type "
4464 << rso.str() + " is unsupported by vectorizer";
4468 Instruction *Inst = dyn_cast<Instruction>(V);
4472 if (Inst->getOpcode() != Opcode0) {
4473 R.getORE()->emit([&]() {
4474 return OptimizationRemarkMissed(
4475 SV_NAME, "InequableTypes", I0)
4476 << "Cannot SLP vectorize list: not all of the "
4477 << "parts of scalar instructions are of the same type: "
4478 << ore::NV("Instruction1Opcode", I0) << " and "
4479 << ore::NV("Instruction2Opcode", Inst);
4485 bool Changed = false;
4486 bool CandidateFound = false;
4487 int MinCost = SLPCostThreshold;
4489 // Keep track of values that were deleted by vectorizing in the loop below.
4490 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4492 unsigned NextInst = 0, MaxInst = VL.size();
4493 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4495 // No actual vectorization should happen, if number of parts is the same as
4496 // provided vectorization factor (i.e. the scalar type is used for vector
4497 // code during codegen).
4498 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4499 if (TTI->getNumberOfParts(VecTy) == VF)
4501 for (unsigned I = NextInst; I < MaxInst; ++I) {
4502 unsigned OpsWidth = 0;
4504 if (I + VF > MaxInst)
4505 OpsWidth = MaxInst - I;
4509 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4512 // Check that a previous iteration of this loop did not delete the Value.
4513 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4516 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4518 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4520 ArrayRef<Value *> EmptyArray;
4521 ArrayRef<Value *> BuildVectorSlice;
4522 if (!BuildVector.empty())
4523 BuildVectorSlice = BuildVector.slice(I, OpsWidth);
4525 R.buildTree(Ops, NeedExtraction ? EmptyArray : BuildVectorSlice);
4526 // TODO: check if we can allow reordering for more cases.
4527 if (AllowReorder && R.shouldReorder()) {
4528 // Conceptually, there is nothing actually preventing us from trying to
4529 // reorder a larger list. In fact, we do exactly this when vectorizing
4530 // reductions. However, at this point, we only expect to get here when
4531 // there are exactly two operations.
4532 assert(Ops.size() == 2);
4533 assert(BuildVectorSlice.empty());
4534 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4535 R.buildTree(ReorderedOps, None);
4537 if (R.isTreeTinyAndNotFullyVectorizable())
4540 R.computeMinimumValueSizes();
4541 int Cost = R.getTreeCost();
4542 CandidateFound = true;
4543 MinCost = std::min(MinCost, Cost);
4545 if (Cost < -SLPCostThreshold) {
4546 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4547 R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4548 cast<Instruction>(Ops[0]))
4549 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4550 << " and with tree size "
4551 << ore::NV("TreeSize", R.getTreeSize()));
4553 Value *VectorizedRoot = R.vectorizeTree();
4555 // Reconstruct the build vector by extracting the vectorized root. This
4556 // way we handle the case where some elements of the vector are
4558 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
4559 if (!BuildVectorSlice.empty()) {
4560 // The insert point is the last build vector instruction. The
4561 // vectorized root will precede it. This guarantees that we get an
4562 // instruction. The vectorized tree could have been constant folded.
4563 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4564 unsigned VecIdx = 0;
4565 for (auto &V : BuildVectorSlice) {
4566 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4567 ++BasicBlock::iterator(InsertAfter));
4568 Instruction *I = cast<Instruction>(V);
4569 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I));
4570 Instruction *Extract =
4571 cast<Instruction>(Builder.CreateExtractElement(
4572 VectorizedRoot, Builder.getInt32(VecIdx++)));
4573 I->setOperand(1, Extract);
4574 I->moveAfter(Extract);
4578 // Move to the next bundle.
4586 if (!Changed && CandidateFound) {
4587 R.getORE()->emit([&]() {
4588 return OptimizationRemarkMissed(
4589 SV_NAME, "NotBeneficial", I0)
4590 << "List vectorization was possible but not beneficial with cost "
4591 << ore::NV("Cost", MinCost) << " >= "
4592 << ore::NV("Treshold", -SLPCostThreshold);
4594 } else if (!Changed) {
4595 R.getORE()->emit([&]() {
4596 return OptimizationRemarkMissed(
4597 SV_NAME, "NotPossible", I0)
4598 << "Cannot SLP vectorize list: vectorization was impossible"
4599 << " with available vectorization factors";
4605 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4609 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4612 Value *P = I->getParent();
4614 // Vectorize in current basic block only.
4615 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4616 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4617 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4620 // Try to vectorize V.
4621 if (tryToVectorizePair(Op0, Op1, R))
4624 auto *A = dyn_cast<BinaryOperator>(Op0);
4625 auto *B = dyn_cast<BinaryOperator>(Op1);
4627 if (B && B->hasOneUse()) {
4628 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4629 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4630 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4632 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4637 if (A && A->hasOneUse()) {
4638 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4639 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4640 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4642 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4648 /// \brief Generate a shuffle mask to be used in a reduction tree.
4650 /// \param VecLen The length of the vector to be reduced.
4651 /// \param NumEltsToRdx The number of elements that should be reduced in the
4653 /// \param IsPairwise Whether the reduction is a pairwise or splitting
4654 /// reduction. A pairwise reduction will generate a mask of
4655 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
4656 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4657 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
4658 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4659 bool IsPairwise, bool IsLeft,
4660 IRBuilder<> &Builder) {
4661 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
4663 SmallVector<Constant *, 32> ShuffleMask(
4664 VecLen, UndefValue::get(Builder.getInt32Ty()));
4667 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4668 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4669 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4671 // Move the upper half of the vector to the lower half.
4672 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4673 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4675 return ConstantVector::get(ShuffleMask);
4680 /// Model horizontal reductions.
4682 /// A horizontal reduction is a tree of reduction operations (currently add and
4683 /// fadd) that has operations that can be put into a vector as its leaf.
4684 /// For example, this tree:
4691 /// This tree has "mul" as its reduced values and "+" as its reduction
4692 /// operations. A reduction might be feeding into a store or a binary operation
4707 class HorizontalReduction {
4708 using ReductionOpsType = SmallVector<Value *, 16>;
4709 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
4710 ReductionOpsListType ReductionOps;
4711 SmallVector<Value *, 32> ReducedVals;
4712 // Use map vector to make stable output.
4713 MapVector<Instruction *, Value *> ExtraArgs;
4715 /// Kind of the reduction data.
4716 enum ReductionKind {
4717 RK_None, /// Not a reduction.
4718 RK_Arithmetic, /// Binary reduction data.
4719 RK_Min, /// Minimum reduction data.
4720 RK_UMin, /// Unsigned minimum reduction data.
4721 RK_Max, /// Maximum reduction data.
4722 RK_UMax, /// Unsigned maximum reduction data.
4725 /// Contains info about operation, like its opcode, left and right operands.
4726 class OperationData {
4727 /// Opcode of the instruction.
4728 unsigned Opcode = 0;
4730 /// Left operand of the reduction operation.
4731 Value *LHS = nullptr;
4733 /// Right operand of the reduction operation.
4734 Value *RHS = nullptr;
4736 /// Kind of the reduction operation.
4737 ReductionKind Kind = RK_None;
4739 /// True if float point min/max reduction has no NaNs.
4742 /// Checks if the reduction operation can be vectorized.
4743 bool isVectorizable() const {
4744 return LHS && RHS &&
4745 // We currently only support adds && min/max reductions.
4746 ((Kind == RK_Arithmetic &&
4747 (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
4748 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
4749 (Kind == RK_Min || Kind == RK_Max)) ||
4750 (Opcode == Instruction::ICmp &&
4751 (Kind == RK_UMin || Kind == RK_UMax)));
4754 /// Creates reduction operation with the current opcode.
4755 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
4756 assert(isVectorizable() &&
4757 "Expected add|fadd or min/max reduction operation.");
4761 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
4764 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
4765 : Builder.CreateFCmpOLT(LHS, RHS);
4768 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
4769 : Builder.CreateFCmpOGT(LHS, RHS);
4772 assert(Opcode == Instruction::ICmp && "Expected integer types.");
4773 Cmp = Builder.CreateICmpULT(LHS, RHS);
4776 assert(Opcode == Instruction::ICmp && "Expected integer types.");
4777 Cmp = Builder.CreateICmpUGT(LHS, RHS);
4780 llvm_unreachable("Unknown reduction operation.");
4782 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
4786 explicit OperationData() = default;
4788 /// Construction for reduced values. They are identified by opcode only and
4789 /// don't have associated LHS/RHS values.
4790 explicit OperationData(Value *V) {
4791 if (auto *I = dyn_cast<Instruction>(V))
4792 Opcode = I->getOpcode();
4795 /// Constructor for reduction operations with opcode and its left and
4797 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
4799 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
4800 assert(Kind != RK_None && "One of the reduction operations is expected.");
4803 explicit operator bool() const { return Opcode; }
4805 /// Get the index of the first operand.
4806 unsigned getFirstOperandIndex() const {
4807 assert(!!*this && "The opcode is not set.");
4821 /// Total number of operands in the reduction operation.
4822 unsigned getNumberOfOperands() const {
4823 assert(Kind != RK_None && !!*this && LHS && RHS &&
4824 "Expected reduction operation.");
4836 llvm_unreachable("Reduction kind is not set");
4839 /// Checks if the operation has the same parent as \p P.
4840 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
4841 assert(Kind != RK_None && !!*this && LHS && RHS &&
4842 "Expected reduction operation.");
4844 return I->getParent() == P;
4847 // Arithmetic reduction operation must be used once only.
4848 return I->getParent() == P;
4853 // SelectInst must be used twice while the condition op must have single
4855 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
4856 return I->getParent() == P && Cmp && Cmp->getParent() == P;
4861 llvm_unreachable("Reduction kind is not set");
4863 /// Expected number of uses for reduction operations/reduced values.
4864 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
4865 assert(Kind != RK_None && !!*this && LHS && RHS &&
4866 "Expected reduction operation.");
4869 return I->hasOneUse();
4874 return I->hasNUses(2) &&
4876 cast<SelectInst>(I)->getCondition()->hasOneUse());
4880 llvm_unreachable("Reduction kind is not set");
4883 /// Initializes the list of reduction operations.
4884 void initReductionOps(ReductionOpsListType &ReductionOps) {
4885 assert(Kind != RK_None && !!*this && LHS && RHS &&
4886 "Expected reduction operation.");
4889 ReductionOps.assign(1, ReductionOpsType());
4895 ReductionOps.assign(2, ReductionOpsType());
4898 llvm_unreachable("Reduction kind is not set");
4901 /// Add all reduction operations for the reduction instruction \p I.
4902 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
4903 assert(Kind != RK_None && !!*this && LHS && RHS &&
4904 "Expected reduction operation.");
4907 ReductionOps[0].emplace_back(I);
4913 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
4914 ReductionOps[1].emplace_back(I);
4917 llvm_unreachable("Reduction kind is not set");
4921 /// Checks if instruction is associative and can be vectorized.
4922 bool isAssociative(Instruction *I) const {
4923 assert(Kind != RK_None && *this && LHS && RHS &&
4924 "Expected reduction operation.");
4927 return I->isAssociative();
4930 return Opcode == Instruction::ICmp ||
4931 cast<Instruction>(I->getOperand(0))->isFast();
4934 assert(Opcode == Instruction::ICmp &&
4935 "Only integer compare operation is expected.");
4940 llvm_unreachable("Reduction kind is not set");
4943 /// Checks if the reduction operation can be vectorized.
4944 bool isVectorizable(Instruction *I) const {
4945 return isVectorizable() && isAssociative(I);
4948 /// Checks if two operation data are both a reduction op or both a reduced
4950 bool operator==(const OperationData &OD) {
4951 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
4952 "One of the comparing operations is incorrect.");
4953 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
4955 bool operator!=(const OperationData &OD) { return !(*this == OD); }
4964 /// Get the opcode of the reduction operation.
4965 unsigned getOpcode() const {
4966 assert(isVectorizable() && "Expected vectorizable operation.");
4970 /// Get kind of reduction data.
4971 ReductionKind getKind() const { return Kind; }
4972 Value *getLHS() const { return LHS; }
4973 Value *getRHS() const { return RHS; }
4974 Type *getConditionType() const {
4982 return CmpInst::makeCmpResultType(LHS->getType());
4986 llvm_unreachable("Reduction kind is not set");
4989 /// Creates reduction operation with the current opcode with the IR flags
4990 /// from \p ReductionOps.
4991 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
4992 const ReductionOpsListType &ReductionOps) const {
4993 assert(isVectorizable() &&
4994 "Expected add|fadd or min/max reduction operation.");
4995 auto *Op = createOp(Builder, Name);
4998 propagateIRFlags(Op, ReductionOps[0]);
5004 if (auto *SI = dyn_cast<SelectInst>(Op))
5005 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5006 propagateIRFlags(Op, ReductionOps[1]);
5011 llvm_unreachable("Unknown reduction operation.");
5013 /// Creates reduction operation with the current opcode with the IR flags
5015 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5016 Instruction *I) const {
5017 assert(isVectorizable() &&
5018 "Expected add|fadd or min/max reduction operation.");
5019 auto *Op = createOp(Builder, Name);
5022 propagateIRFlags(Op, I);
5028 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5029 propagateIRFlags(SI->getCondition(),
5030 cast<SelectInst>(I)->getCondition());
5032 propagateIRFlags(Op, I);
5037 llvm_unreachable("Unknown reduction operation.");
5040 TargetTransformInfo::ReductionFlags getFlags() const {
5041 TargetTransformInfo::ReductionFlags Flags;
5042 Flags.NoNaN = NoNaN;
5047 Flags.IsSigned = Opcode == Instruction::ICmp;
5048 Flags.IsMaxOp = false;
5051 Flags.IsSigned = Opcode == Instruction::ICmp;
5052 Flags.IsMaxOp = true;
5055 Flags.IsSigned = false;
5056 Flags.IsMaxOp = false;
5059 Flags.IsSigned = false;
5060 Flags.IsMaxOp = true;
5063 llvm_unreachable("Reduction kind is not set");
5069 Instruction *ReductionRoot = nullptr;
5071 /// The operation data of the reduction operation.
5072 OperationData ReductionData;
5074 /// The operation data of the values we perform a reduction on.
5075 OperationData ReducedValueData;
5077 /// Should we model this reduction as a pairwise reduction tree or a tree that
5078 /// splits the vector in halves and adds those halves.
5079 bool IsPairwiseReduction = false;
5081 /// Checks if the ParentStackElem.first should be marked as a reduction
5082 /// operation with an extra argument or as extra argument itself.
5083 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5085 if (ExtraArgs.count(ParentStackElem.first)) {
5086 ExtraArgs[ParentStackElem.first] = nullptr;
5087 // We ran into something like:
5088 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5089 // The whole ParentStackElem.first should be considered as an extra value
5091 // Do not perform analysis of remaining operands of ParentStackElem.first
5092 // instruction, this whole instruction is an extra argument.
5093 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5095 // We ran into something like:
5096 // ParentStackElem.first += ... + ExtraArg + ...
5097 ExtraArgs[ParentStackElem.first] = ExtraArg;
5101 static OperationData getOperationData(Value *V) {
5103 return OperationData();
5107 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5108 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5111 if (auto *Select = dyn_cast<SelectInst>(V)) {
5112 // Look for a min/max pattern.
5113 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5114 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5115 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5116 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5117 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5118 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5119 return OperationData(
5120 Instruction::FCmp, LHS, RHS, RK_Min,
5121 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5122 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5123 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5124 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5125 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5126 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5127 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5128 return OperationData(
5129 Instruction::FCmp, LHS, RHS, RK_Max,
5130 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5133 return OperationData(V);
5137 HorizontalReduction() = default;
5139 /// \brief Try to find a reduction tree.
5140 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5141 assert((!Phi || is_contained(Phi->operands(), B)) &&
5142 "Thi phi needs to use the binary operator");
5144 ReductionData = getOperationData(B);
5146 // We could have a initial reductions that is not an add.
5147 // r *= v1 + v2 + v3 + v4
5148 // In such a case start looking for a tree rooted in the first '+'.
5150 if (ReductionData.getLHS() == Phi) {
5152 B = dyn_cast<Instruction>(ReductionData.getRHS());
5153 ReductionData = getOperationData(B);
5154 } else if (ReductionData.getRHS() == Phi) {
5156 B = dyn_cast<Instruction>(ReductionData.getLHS());
5157 ReductionData = getOperationData(B);
5161 if (!ReductionData.isVectorizable(B))
5164 Type *Ty = B->getType();
5165 if (!isValidElementType(Ty))
5168 ReducedValueData.clear();
5171 // Post order traverse the reduction tree starting at B. We only handle true
5172 // trees containing only binary operators.
5173 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5174 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5175 ReductionData.initReductionOps(ReductionOps);
5176 while (!Stack.empty()) {
5177 Instruction *TreeN = Stack.back().first;
5178 unsigned EdgeToVist = Stack.back().second++;
5179 OperationData OpData = getOperationData(TreeN);
5180 bool IsReducedValue = OpData != ReductionData;
5183 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5185 ReducedVals.push_back(TreeN);
5187 auto I = ExtraArgs.find(TreeN);
5188 if (I != ExtraArgs.end() && !I->second) {
5189 // Check if TreeN is an extra argument of its parent operation.
5190 if (Stack.size() <= 1) {
5191 // TreeN can't be an extra argument as it is a root reduction
5195 // Yes, TreeN is an extra argument, do not add it to a list of
5196 // reduction operations.
5197 // Stack[Stack.size() - 2] always points to the parent operation.
5198 markExtraArg(Stack[Stack.size() - 2], TreeN);
5199 ExtraArgs.erase(TreeN);
5201 ReductionData.addReductionOps(TreeN, ReductionOps);
5208 // Visit left or right.
5209 Value *NextV = TreeN->getOperand(EdgeToVist);
5211 auto *I = dyn_cast<Instruction>(NextV);
5212 OpData = getOperationData(I);
5213 // Continue analysis if the next operand is a reduction operation or
5214 // (possibly) a reduced value. If the reduced value opcode is not set,
5215 // the first met operation != reduction operation is considered as the
5216 // reduced value class.
5217 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5218 OpData == ReductionData)) {
5219 const bool IsReductionOperation = OpData == ReductionData;
5220 // Only handle trees in the current basic block.
5221 if (!ReductionData.hasSameParent(I, B->getParent(),
5222 IsReductionOperation)) {
5223 // I is an extra argument for TreeN (its parent operation).
5224 markExtraArg(Stack.back(), I);
5228 // Each tree node needs to have minimal number of users except for the
5229 // ultimate reduction.
5230 if (!ReductionData.hasRequiredNumberOfUses(I,
5231 OpData == ReductionData) &&
5233 // I is an extra argument for TreeN (its parent operation).
5234 markExtraArg(Stack.back(), I);
5238 if (IsReductionOperation) {
5239 // We need to be able to reassociate the reduction operations.
5240 if (!OpData.isAssociative(I)) {
5241 // I is an extra argument for TreeN (its parent operation).
5242 markExtraArg(Stack.back(), I);
5245 } else if (ReducedValueData &&
5246 ReducedValueData != OpData) {
5247 // Make sure that the opcodes of the operations that we are going to
5249 // I is an extra argument for TreeN (its parent operation).
5250 markExtraArg(Stack.back(), I);
5252 } else if (!ReducedValueData)
5253 ReducedValueData = OpData;
5255 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5259 // NextV is an extra argument for TreeN (its parent operation).
5260 markExtraArg(Stack.back(), NextV);
5265 /// \brief Attempt to vectorize the tree found by
5266 /// matchAssociativeReduction.
5267 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5268 if (ReducedVals.empty())
5271 // If there is a sufficient number of reduction values, reduce
5272 // to a nearby power-of-2. Can safely generate oversized
5273 // vectors and rely on the backend to split them to legal sizes.
5274 unsigned NumReducedVals = ReducedVals.size();
5275 if (NumReducedVals < 4)
5278 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5280 Value *VectorizedTree = nullptr;
5281 IRBuilder<> Builder(ReductionRoot);
5282 FastMathFlags Unsafe;
5284 Builder.setFastMathFlags(Unsafe);
5287 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5288 // The same extra argument may be used several time, so log each attempt
5290 for (auto &Pair : ExtraArgs)
5291 ExternallyUsedValues[Pair.second].push_back(Pair.first);
5292 SmallVector<Value *, 16> IgnoreList;
5293 for (auto &V : ReductionOps)
5294 IgnoreList.append(V.begin(), V.end());
5295 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5296 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5297 V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5298 if (V.shouldReorder()) {
5299 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
5300 V.buildTree(Reversed, ExternallyUsedValues, IgnoreList);
5302 if (V.isTreeTinyAndNotFullyVectorizable())
5305 V.computeMinimumValueSizes();
5309 V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5310 if (Cost >= -SLPCostThreshold) {
5311 V.getORE()->emit([&]() {
5312 return OptimizationRemarkMissed(
5313 SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5314 << "Vectorizing horizontal reduction is possible"
5315 << "but not beneficial with cost "
5316 << ore::NV("Cost", Cost) << " and threshold "
5317 << ore::NV("Threshold", -SLPCostThreshold);
5322 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Cost
5324 V.getORE()->emit([&]() {
5325 return OptimizationRemark(
5326 SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5327 << "Vectorized horizontal reduction with cost "
5328 << ore::NV("Cost", Cost) << " and with tree size "
5329 << ore::NV("TreeSize", V.getTreeSize());
5332 // Vectorize a tree.
5333 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5334 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5336 // Emit a reduction.
5337 Value *ReducedSubTree =
5338 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5339 if (VectorizedTree) {
5340 Builder.SetCurrentDebugLocation(Loc);
5341 OperationData VectReductionData(ReductionData.getOpcode(),
5342 VectorizedTree, ReducedSubTree,
5343 ReductionData.getKind());
5345 VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5347 VectorizedTree = ReducedSubTree;
5349 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5352 if (VectorizedTree) {
5353 // Finish the reduction.
5354 for (; i < NumReducedVals; ++i) {
5355 auto *I = cast<Instruction>(ReducedVals[i]);
5356 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5357 OperationData VectReductionData(ReductionData.getOpcode(),
5359 ReductionData.getKind());
5360 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5362 for (auto &Pair : ExternallyUsedValues) {
5363 assert(!Pair.second.empty() &&
5364 "At least one DebugLoc must be inserted");
5365 // Add each externally used value to the final reduction.
5366 for (auto *I : Pair.second) {
5367 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5368 OperationData VectReductionData(ReductionData.getOpcode(),
5369 VectorizedTree, Pair.first,
5370 ReductionData.getKind());
5371 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5375 ReductionRoot->replaceAllUsesWith(VectorizedTree);
5377 return VectorizedTree != nullptr;
5380 unsigned numReductionValues() const {
5381 return ReducedVals.size();
5385 /// \brief Calculate the cost of a reduction.
5386 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5387 unsigned ReduxWidth) {
5388 Type *ScalarTy = FirstReducedVal->getType();
5389 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5391 int PairwiseRdxCost;
5392 int SplittingRdxCost;
5393 switch (ReductionData.getKind()) {
5396 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5397 /*IsPairwiseForm=*/true);
5399 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5400 /*IsPairwiseForm=*/false);
5406 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5407 bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5408 ReductionData.getKind() == RK_UMax;
5410 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5411 /*IsPairwiseForm=*/true, IsUnsigned);
5413 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5414 /*IsPairwiseForm=*/false, IsUnsigned);
5418 llvm_unreachable("Expected arithmetic or min/max reduction operation");
5421 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5422 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5424 int ScalarReduxCost;
5425 switch (ReductionData.getKind()) {
5428 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5435 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5436 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5437 CmpInst::makeCmpResultType(ScalarTy));
5440 llvm_unreachable("Expected arithmetic or min/max reduction operation");
5442 ScalarReduxCost *= (ReduxWidth - 1);
5444 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost
5445 << " for reduction that starts with " << *FirstReducedVal
5447 << (IsPairwiseReduction ? "pairwise" : "splitting")
5448 << " reduction)\n");
5450 return VecReduxCost - ScalarReduxCost;
5453 /// \brief Emit a horizontal reduction of the vectorized value.
5454 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5455 unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5456 assert(VectorizedValue && "Need to have a vectorized tree node");
5457 assert(isPowerOf2_32(ReduxWidth) &&
5458 "We only handle power-of-two reductions for now");
5460 if (!IsPairwiseReduction)
5461 return createSimpleTargetReduction(
5462 Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5463 ReductionData.getFlags(), ReductionOps.back());
5465 Value *TmpVec = VectorizedValue;
5466 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5468 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5470 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5472 Value *LeftShuf = Builder.CreateShuffleVector(
5473 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5474 Value *RightShuf = Builder.CreateShuffleVector(
5475 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5477 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5478 RightShuf, ReductionData.getKind());
5479 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5482 // The result is in the first element of the vector.
5483 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5487 } // end anonymous namespace
5489 /// \brief Recognize construction of vectors like
5490 /// %ra = insertelement <4 x float> undef, float %s0, i32 0
5491 /// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5492 /// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5493 /// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5494 /// starting from the last insertelement instruction.
5496 /// Returns true if it matches
5497 static bool findBuildVector(InsertElementInst *LastInsertElem,
5498 SmallVectorImpl<Value *> &BuildVector,
5499 SmallVectorImpl<Value *> &BuildVectorOpds) {
5502 BuildVector.push_back(LastInsertElem);
5503 BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5504 V = LastInsertElem->getOperand(0);
5505 if (isa<UndefValue>(V))
5507 LastInsertElem = dyn_cast<InsertElementInst>(V);
5508 if (!LastInsertElem || !LastInsertElem->hasOneUse())
5511 std::reverse(BuildVector.begin(), BuildVector.end());
5512 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5516 /// \brief Like findBuildVector, but looks for construction of aggregate.
5518 /// \return true if it matches.
5519 static bool findBuildAggregate(InsertValueInst *IV,
5520 SmallVectorImpl<Value *> &BuildVector,
5521 SmallVectorImpl<Value *> &BuildVectorOpds) {
5524 BuildVector.push_back(IV);
5525 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
5526 V = IV->getAggregateOperand();
5527 if (isa<UndefValue>(V))
5529 IV = dyn_cast<InsertValueInst>(V);
5530 if (!IV || !IV->hasOneUse())
5533 std::reverse(BuildVector.begin(), BuildVector.end());
5534 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5538 static bool PhiTypeSorterFunc(Value *V, Value *V2) {
5539 return V->getType() < V2->getType();
5542 /// \brief Try and get a reduction value from a phi node.
5544 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions
5545 /// if they come from either \p ParentBB or a containing loop latch.
5547 /// \returns A candidate reduction value if possible, or \code nullptr \endcode
5548 /// if not possible.
5549 static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
5550 BasicBlock *ParentBB, LoopInfo *LI) {
5551 // There are situations where the reduction value is not dominated by the
5552 // reduction phi. Vectorizing such cases has been reported to cause
5553 // miscompiles. See PR25787.
5554 auto DominatedReduxValue = [&](Value *R) {
5556 dyn_cast<Instruction>(R) &&
5557 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
5560 Value *Rdx = nullptr;
5562 // Return the incoming value if it comes from the same BB as the phi node.
5563 if (P->getIncomingBlock(0) == ParentBB) {
5564 Rdx = P->getIncomingValue(0);
5565 } else if (P->getIncomingBlock(1) == ParentBB) {
5566 Rdx = P->getIncomingValue(1);
5569 if (Rdx && DominatedReduxValue(Rdx))
5572 // Otherwise, check whether we have a loop latch to look at.
5573 Loop *BBL = LI->getLoopFor(ParentBB);
5576 BasicBlock *BBLatch = BBL->getLoopLatch();
5580 // There is a loop latch, return the incoming value if it comes from
5581 // that. This reduction pattern occasionally turns up.
5582 if (P->getIncomingBlock(0) == BBLatch) {
5583 Rdx = P->getIncomingValue(0);
5584 } else if (P->getIncomingBlock(1) == BBLatch) {
5585 Rdx = P->getIncomingValue(1);
5588 if (Rdx && DominatedReduxValue(Rdx))
5594 /// Attempt to reduce a horizontal reduction.
5595 /// If it is legal to match a horizontal reduction feeding the phi node \a P
5596 /// with reduction operators \a Root (or one of its operands) in a basic block
5597 /// \a BB, then check if it can be done. If horizontal reduction is not found
5598 /// and root instruction is a binary operation, vectorization of the operands is
5600 /// \returns true if a horizontal reduction was matched and reduced or operands
5601 /// of one of the binary instruction were vectorized.
5602 /// \returns false if a horizontal reduction was not matched (or not possible)
5603 /// or no vectorization of any binary operation feeding \a Root instruction was
5605 static bool tryToVectorizeHorReductionOrInstOperands(
5606 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
5607 TargetTransformInfo *TTI,
5608 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
5609 if (!ShouldVectorizeHor)
5615 if (Root->getParent() != BB || isa<PHINode>(Root))
5617 // Start analysis starting from Root instruction. If horizontal reduction is
5618 // found, try to vectorize it. If it is not a horizontal reduction or
5619 // vectorization is not possible or not effective, and currently analyzed
5620 // instruction is a binary operation, try to vectorize the operands, using
5621 // pre-order DFS traversal order. If the operands were not vectorized, repeat
5622 // the same procedure considering each operand as a possible root of the
5623 // horizontal reduction.
5624 // Interrupt the process if the Root instruction itself was vectorized or all
5625 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
5626 SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
5627 SmallSet<Value *, 8> VisitedInstrs;
5629 while (!Stack.empty()) {
5632 std::tie(V, Level) = Stack.pop_back_val();
5635 auto *Inst = dyn_cast<Instruction>(V);
5638 auto *BI = dyn_cast<BinaryOperator>(Inst);
5639 auto *SI = dyn_cast<SelectInst>(Inst);
5641 HorizontalReduction HorRdx;
5642 if (HorRdx.matchAssociativeReduction(P, Inst)) {
5643 if (HorRdx.tryToReduce(R, TTI)) {
5645 // Set P to nullptr to avoid re-analysis of phi node in
5646 // matchAssociativeReduction function unless this is the root node.
5652 Inst = dyn_cast<Instruction>(BI->getOperand(0));
5654 Inst = dyn_cast<Instruction>(BI->getOperand(1));
5656 // Set P to nullptr to avoid re-analysis of phi node in
5657 // matchAssociativeReduction function unless this is the root node.
5663 // Set P to nullptr to avoid re-analysis of phi node in
5664 // matchAssociativeReduction function unless this is the root node.
5666 if (Vectorize(Inst, R)) {
5671 // Try to vectorize operands.
5672 // Continue analysis for the instruction from the same basic block only to
5673 // save compile time.
5674 if (++Level < RecursionMaxDepth)
5675 for (auto *Op : Inst->operand_values())
5676 if (VisitedInstrs.insert(Op).second)
5677 if (auto *I = dyn_cast<Instruction>(Op))
5678 if (!isa<PHINode>(I) && I->getParent() == BB)
5679 Stack.emplace_back(Op, Level);
5684 bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
5685 BasicBlock *BB, BoUpSLP &R,
5686 TargetTransformInfo *TTI) {
5689 auto *I = dyn_cast<Instruction>(V);
5693 if (!isa<BinaryOperator>(I))
5695 // Try to match and vectorize a horizontal reduction.
5696 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
5697 return tryToVectorize(I, R);
5699 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
5700 ExtraVectorization);
5703 bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
5704 BasicBlock *BB, BoUpSLP &R) {
5705 const DataLayout &DL = BB->getModule()->getDataLayout();
5706 if (!R.canMapToVector(IVI->getType(), DL))
5709 SmallVector<Value *, 16> BuildVector;
5710 SmallVector<Value *, 16> BuildVectorOpds;
5711 if (!findBuildAggregate(IVI, BuildVector, BuildVectorOpds))
5714 DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
5715 // Aggregate value is unlikely to be processed in vector register, we need to
5716 // extract scalars into scalar registers, so NeedExtraction is set true.
5717 return tryToVectorizeList(BuildVectorOpds, R, BuildVector, false, true);
5720 bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
5721 BasicBlock *BB, BoUpSLP &R) {
5722 SmallVector<Value *, 16> BuildVector;
5723 SmallVector<Value *, 16> BuildVectorOpds;
5724 if (!findBuildVector(IEI, BuildVector, BuildVectorOpds))
5727 // Vectorize starting with the build vector operands ignoring the BuildVector
5728 // instructions for the purpose of scheduling and user extraction.
5729 return tryToVectorizeList(BuildVectorOpds, R, BuildVector);
5732 bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
5734 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
5737 bool OpsChanged = false;
5738 for (int Idx = 0; Idx < 2; ++Idx) {
5740 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
5745 bool SLPVectorizerPass::vectorizeSimpleInstructions(
5746 SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
5747 bool OpsChanged = false;
5748 for (auto &VH : reverse(Instructions)) {
5749 auto *I = dyn_cast_or_null<Instruction>(VH);
5752 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
5753 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
5754 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
5755 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
5756 else if (auto *CI = dyn_cast<CmpInst>(I))
5757 OpsChanged |= vectorizeCmpInst(CI, BB, R);
5759 Instructions.clear();
5763 bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
5764 bool Changed = false;
5765 SmallVector<Value *, 4> Incoming;
5766 SmallSet<Value *, 16> VisitedInstrs;
5768 bool HaveVectorizedPhiNodes = true;
5769 while (HaveVectorizedPhiNodes) {
5770 HaveVectorizedPhiNodes = false;
5772 // Collect the incoming values from the PHIs.
5774 for (Instruction &I : *BB) {
5775 PHINode *P = dyn_cast<PHINode>(&I);
5779 if (!VisitedInstrs.count(P))
5780 Incoming.push_back(P);
5784 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
5786 // Try to vectorize elements base on their type.
5787 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
5791 // Look for the next elements with the same type.
5792 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
5793 while (SameTypeIt != E &&
5794 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
5795 VisitedInstrs.insert(*SameTypeIt);
5799 // Try to vectorize them.
5800 unsigned NumElts = (SameTypeIt - IncIt);
5801 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n");
5802 // The order in which the phi nodes appear in the program does not matter.
5803 // So allow tryToVectorizeList to reorder them if it is beneficial. This
5804 // is done when there are exactly two elements since tryToVectorizeList
5805 // asserts that there are only two values when AllowReorder is true.
5806 bool AllowReorder = NumElts == 2;
5807 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
5808 None, AllowReorder)) {
5809 // Success start over because instructions might have been changed.
5810 HaveVectorizedPhiNodes = true;
5815 // Start over at the next instruction of a different type (or the end).
5820 VisitedInstrs.clear();
5822 SmallVector<WeakVH, 8> PostProcessInstructions;
5823 SmallDenseSet<Instruction *, 4> KeyNodes;
5824 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
5825 // We may go through BB multiple times so skip the one we have checked.
5826 if (!VisitedInstrs.insert(&*it).second) {
5827 if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
5828 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
5829 // We would like to start over since some instructions are deleted
5830 // and the iterator may become invalid value.
5838 if (isa<DbgInfoIntrinsic>(it))
5841 // Try to vectorize reductions that use PHINodes.
5842 if (PHINode *P = dyn_cast<PHINode>(it)) {
5843 // Check that the PHI is a reduction PHI.
5844 if (P->getNumIncomingValues() != 2)
5847 // Try to match and vectorize a horizontal reduction.
5848 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
5858 // Ran into an instruction without users, like terminator, or function call
5859 // with ignored return value, store. Ignore unused instructions (basing on
5860 // instruction type, except for CallInst and InvokeInst).
5861 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
5862 isa<InvokeInst>(it))) {
5863 KeyNodes.insert(&*it);
5864 bool OpsChanged = false;
5865 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
5866 for (auto *V : it->operand_values()) {
5867 // Try to match and vectorize a horizontal reduction.
5868 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
5871 // Start vectorization of post-process list of instructions from the
5872 // top-tree instructions to try to vectorize as many instructions as
5874 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
5876 // We would like to start over since some instructions are deleted
5877 // and the iterator may become invalid value.
5885 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
5886 isa<InsertValueInst>(it))
5887 PostProcessInstructions.push_back(&*it);
5894 bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
5895 auto Changed = false;
5896 for (auto &Entry : GEPs) {
5897 // If the getelementptr list has fewer than two elements, there's nothing
5899 if (Entry.second.size() < 2)
5902 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "
5903 << Entry.second.size() << ".\n");
5905 // We process the getelementptr list in chunks of 16 (like we do for
5906 // stores) to minimize compile-time.
5907 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
5908 auto Len = std::min<unsigned>(BE - BI, 16);
5909 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
5911 // Initialize a set a candidate getelementptrs. Note that we use a
5912 // SetVector here to preserve program order. If the index computations
5913 // are vectorizable and begin with loads, we want to minimize the chance
5914 // of having to reorder them later.
5915 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
5917 // Some of the candidates may have already been vectorized after we
5918 // initially collected them. If so, the WeakTrackingVHs will have
5920 // values, so remove them from the set of candidates.
5921 Candidates.remove(nullptr);
5923 // Remove from the set of candidates all pairs of getelementptrs with
5924 // constant differences. Such getelementptrs are likely not good
5925 // candidates for vectorization in a bottom-up phase since one can be
5926 // computed from the other. We also ensure all candidate getelementptr
5927 // indices are unique.
5928 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
5929 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
5930 if (!Candidates.count(GEPI))
5932 auto *SCEVI = SE->getSCEV(GEPList[I]);
5933 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
5934 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
5935 auto *SCEVJ = SE->getSCEV(GEPList[J]);
5936 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
5937 Candidates.remove(GEPList[I]);
5938 Candidates.remove(GEPList[J]);
5939 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
5940 Candidates.remove(GEPList[J]);
5945 // We break out of the above computation as soon as we know there are
5946 // fewer than two candidates remaining.
5947 if (Candidates.size() < 2)
5950 // Add the single, non-constant index of each candidate to the bundle. We
5951 // ensured the indices met these constraints when we originally collected
5952 // the getelementptrs.
5953 SmallVector<Value *, 16> Bundle(Candidates.size());
5954 auto BundleIndex = 0u;
5955 for (auto *V : Candidates) {
5956 auto *GEP = cast<GetElementPtrInst>(V);
5957 auto *GEPIdx = GEP->idx_begin()->get();
5958 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx));
5959 Bundle[BundleIndex++] = GEPIdx;
5962 // Try and vectorize the indices. We are currently only interested in
5963 // gather-like cases of the form:
5965 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
5967 // where the loads of "a", the loads of "b", and the subtractions can be
5968 // performed in parallel. It's likely that detecting this pattern in a
5969 // bottom-up phase will be simpler and less costly than building a
5970 // full-blown top-down phase beginning at the consecutive loads.
5971 Changed |= tryToVectorizeList(Bundle, R);
5977 bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
5978 bool Changed = false;
5979 // Attempt to sort and vectorize each of the store-groups.
5980 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
5982 if (it->second.size() < 2)
5985 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "
5986 << it->second.size() << ".\n");
5988 // Process the stores in chunks of 16.
5989 // TODO: The limit of 16 inhibits greater vectorization factors.
5990 // For example, AVX2 supports v32i8. Increasing this limit, however,
5991 // may cause a significant compile-time increase.
5992 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
5993 unsigned Len = std::min<unsigned>(CE - CI, 16);
5994 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
6000 char SLPVectorizer::ID = 0;
6002 static const char lv_name[] = "SLP Vectorizer";
6004 INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)
6005 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
6006 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
6007 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6008 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
6009 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
6010 INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)
6011 INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
6012 INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)
6014 Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); }