1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
9 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10 // stores that can be put together into vector-stores. Next, it attempts to
11 // construct vectorizable tree using the use-def chains. If a profitable tree
12 // was found, the SLP vectorizer performs vectorization on the tree.
14 // The pass is inspired by the work described in the paper:
15 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17 //===----------------------------------------------------------------------===//
18 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/PostOrderIterator.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/CodeMetrics.h"
24 #include "llvm/Analysis/GlobalsModRef.h"
25 #include "llvm/Analysis/LoopAccessAnalysis.h"
26 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
27 #include "llvm/Analysis/ValueTracking.h"
28 #include "llvm/Analysis/VectorUtils.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/IRBuilder.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/IntrinsicInst.h"
34 #include "llvm/IR/Module.h"
35 #include "llvm/IR/NoFolder.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/IR/Value.h"
38 #include "llvm/IR/Verifier.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/CommandLine.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/raw_ostream.h"
43 #include "llvm/Transforms/Vectorize.h"
48 using namespace slpvectorizer;
50 #define SV_NAME "slp-vectorizer"
51 #define DEBUG_TYPE "SLP"
53 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
56 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
57 cl::desc("Only vectorize if you gain more than this "
61 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
62 cl::desc("Attempt to vectorize horizontal reductions"));
64 static cl::opt<bool> ShouldStartVectorizeHorAtStore(
65 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
67 "Attempt to vectorize horizontal reductions feeding into a store"));
70 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
71 cl::desc("Attempt to vectorize for this register size in bits"));
73 /// Limits the size of scheduling regions in a block.
74 /// It avoid long compile times for _very_ large blocks where vector
75 /// instructions are spread over a wide range.
76 /// This limit is way higher than needed by real-world functions.
78 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
79 cl::desc("Limit the size of the SLP scheduling region per block"));
81 static cl::opt<int> MinVectorRegSizeOption(
82 "slp-min-reg-size", cl::init(128), cl::Hidden,
83 cl::desc("Attempt to vectorize for this register size in bits"));
85 static cl::opt<unsigned> RecursionMaxDepth(
86 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
87 cl::desc("Limit the recursion depth when building a vectorizable tree"));
89 static cl::opt<unsigned> MinTreeSize(
90 "slp-min-tree-size", cl::init(3), cl::Hidden,
91 cl::desc("Only vectorize small trees if they are fully vectorizable"));
93 // Limit the number of alias checks. The limit is chosen so that
94 // it has no negative effect on the llvm benchmarks.
95 static const unsigned AliasedCheckLimit = 10;
97 // Another limit for the alias checks: The maximum distance between load/store
98 // instructions where alias checks are done.
99 // This limit is useful for very large basic blocks.
100 static const unsigned MaxMemDepDistance = 160;
102 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
103 /// regions to be handled.
104 static const int MinScheduleRegionSize = 16;
106 /// \brief Predicate for the element types that the SLP vectorizer supports.
108 /// The most important thing to filter here are types which are invalid in LLVM
109 /// vectors. We also filter target specific types which have absolutely no
110 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
111 /// avoids spending time checking the cost model and realizing that they will
112 /// be inevitably scalarized.
113 static bool isValidElementType(Type *Ty) {
114 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
115 !Ty->isPPC_FP128Ty();
118 /// \returns the parent basic block if all of the instructions in \p VL
119 /// are in the same block or null otherwise.
120 static BasicBlock *getSameBlock(ArrayRef<Value *> VL) {
121 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
124 BasicBlock *BB = I0->getParent();
125 for (int i = 1, e = VL.size(); i < e; i++) {
126 Instruction *I = dyn_cast<Instruction>(VL[i]);
130 if (BB != I->getParent())
136 /// \returns True if all of the values in \p VL are constants.
137 static bool allConstant(ArrayRef<Value *> VL) {
139 if (!isa<Constant>(i))
144 /// \returns True if all of the values in \p VL are identical.
145 static bool isSplat(ArrayRef<Value *> VL) {
146 for (unsigned i = 1, e = VL.size(); i < e; ++i)
152 ///\returns Opcode that can be clubbed with \p Op to create an alternate
153 /// sequence which can later be merged as a ShuffleVector instruction.
154 static unsigned getAltOpcode(unsigned Op) {
156 case Instruction::FAdd:
157 return Instruction::FSub;
158 case Instruction::FSub:
159 return Instruction::FAdd;
160 case Instruction::Add:
161 return Instruction::Sub;
162 case Instruction::Sub:
163 return Instruction::Add;
169 ///\returns bool representing if Opcode \p Op can be part
170 /// of an alternate sequence which can later be merged as
171 /// a ShuffleVector instruction.
172 static bool canCombineAsAltInst(unsigned Op) {
173 return Op == Instruction::FAdd || Op == Instruction::FSub ||
174 Op == Instruction::Sub || Op == Instruction::Add;
177 /// \returns ShuffleVector instruction if instructions in \p VL have
178 /// alternate fadd,fsub / fsub,fadd/add,sub/sub,add sequence.
179 /// (i.e. e.g. opcodes of fadd,fsub,fadd,fsub...)
180 static unsigned isAltInst(ArrayRef<Value *> VL) {
181 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
182 unsigned Opcode = I0->getOpcode();
183 unsigned AltOpcode = getAltOpcode(Opcode);
184 for (int i = 1, e = VL.size(); i < e; i++) {
185 Instruction *I = dyn_cast<Instruction>(VL[i]);
186 if (!I || I->getOpcode() != ((i & 1) ? AltOpcode : Opcode))
189 return Instruction::ShuffleVector;
192 /// \returns The opcode if all of the Instructions in \p VL have the same
194 static unsigned getSameOpcode(ArrayRef<Value *> VL) {
195 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
198 unsigned Opcode = I0->getOpcode();
199 for (int i = 1, e = VL.size(); i < e; i++) {
200 Instruction *I = dyn_cast<Instruction>(VL[i]);
201 if (!I || Opcode != I->getOpcode()) {
202 if (canCombineAsAltInst(Opcode) && i == 1)
203 return isAltInst(VL);
210 /// Get the intersection (logical and) of all of the potential IR flags
211 /// of each scalar operation (VL) that will be converted into a vector (I).
212 /// Flag set: NSW, NUW, exact, and all of fast-math.
213 static void propagateIRFlags(Value *I, ArrayRef<Value *> VL) {
214 if (auto *VecOp = dyn_cast<BinaryOperator>(I)) {
215 if (auto *Intersection = dyn_cast<BinaryOperator>(VL[0])) {
216 // Intersection is initialized to the 0th scalar,
217 // so start counting from index '1'.
218 for (int i = 1, e = VL.size(); i < e; ++i) {
219 if (auto *Scalar = dyn_cast<BinaryOperator>(VL[i]))
220 Intersection->andIRFlags(Scalar);
222 VecOp->copyIRFlags(Intersection);
227 /// \returns The type that all of the values in \p VL have or null if there
228 /// are different types.
229 static Type* getSameType(ArrayRef<Value *> VL) {
230 Type *Ty = VL[0]->getType();
231 for (int i = 1, e = VL.size(); i < e; i++)
232 if (VL[i]->getType() != Ty)
238 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
239 static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
240 assert(Opcode == Instruction::ExtractElement ||
241 Opcode == Instruction::ExtractValue);
242 if (Opcode == Instruction::ExtractElement) {
243 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
244 return CI && CI->getZExtValue() == Idx;
246 ExtractValueInst *EI = cast<ExtractValueInst>(E);
247 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
251 /// \returns True if in-tree use also needs extract. This refers to
252 /// possible scalar operand in vectorized instruction.
253 static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
254 TargetLibraryInfo *TLI) {
256 unsigned Opcode = UserInst->getOpcode();
258 case Instruction::Load: {
259 LoadInst *LI = cast<LoadInst>(UserInst);
260 return (LI->getPointerOperand() == Scalar);
262 case Instruction::Store: {
263 StoreInst *SI = cast<StoreInst>(UserInst);
264 return (SI->getPointerOperand() == Scalar);
266 case Instruction::Call: {
267 CallInst *CI = cast<CallInst>(UserInst);
268 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
269 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
270 return (CI->getArgOperand(1) == Scalar);
278 /// \returns the AA location that is being access by the instruction.
279 static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
280 if (StoreInst *SI = dyn_cast<StoreInst>(I))
281 return MemoryLocation::get(SI);
282 if (LoadInst *LI = dyn_cast<LoadInst>(I))
283 return MemoryLocation::get(LI);
284 return MemoryLocation();
287 /// \returns True if the instruction is not a volatile or atomic load/store.
288 static bool isSimple(Instruction *I) {
289 if (LoadInst *LI = dyn_cast<LoadInst>(I))
290 return LI->isSimple();
291 if (StoreInst *SI = dyn_cast<StoreInst>(I))
292 return SI->isSimple();
293 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
294 return !MI->isVolatile();
299 namespace slpvectorizer {
300 /// Bottom Up SLP Vectorizer.
303 typedef SmallVector<Value *, 8> ValueList;
304 typedef SmallVector<Instruction *, 16> InstrList;
305 typedef SmallPtrSet<Value *, 16> ValueSet;
306 typedef SmallVector<StoreInst *, 8> StoreList;
308 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
309 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
310 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
311 const DataLayout *DL)
312 : NumLoadsWantToKeepOrder(0), NumLoadsWantToChangeOrder(0), F(Func),
313 SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC), DB(DB),
314 DL(DL), Builder(Se->getContext()) {
315 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
316 // Use the vector register size specified by the target unless overridden
317 // by a command-line option.
318 // TODO: It would be better to limit the vectorization factor based on
319 // data type rather than just register size. For example, x86 AVX has
320 // 256-bit registers, but it does not support integer operations
321 // at that width (that requires AVX2).
322 if (MaxVectorRegSizeOption.getNumOccurrences())
323 MaxVecRegSize = MaxVectorRegSizeOption;
325 MaxVecRegSize = TTI->getRegisterBitWidth(true);
327 MinVecRegSize = MinVectorRegSizeOption;
330 /// \brief Vectorize the tree that starts with the elements in \p VL.
331 /// Returns the vectorized root.
332 Value *vectorizeTree();
334 /// \returns the cost incurred by unwanted spills and fills, caused by
335 /// holding live values over call sites.
338 /// \returns the vectorization cost of the subtree that starts at \p VL.
339 /// A negative number means that this is profitable.
342 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
343 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
344 void buildTree(ArrayRef<Value *> Roots,
345 ArrayRef<Value *> UserIgnoreLst = None);
347 /// Clear the internal data structures that are created by 'buildTree'.
349 VectorizableTree.clear();
350 ScalarToTreeEntry.clear();
352 ExternalUses.clear();
353 NumLoadsWantToKeepOrder = 0;
354 NumLoadsWantToChangeOrder = 0;
355 for (auto &Iter : BlocksSchedules) {
356 BlockScheduling *BS = Iter.second.get();
362 /// \brief Perform LICM and CSE on the newly generated gather sequences.
363 void optimizeGatherSequence();
365 /// \returns true if it is beneficial to reverse the vector order.
366 bool shouldReorder() const {
367 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
370 /// \return The vector element size in bits to use when vectorizing the
371 /// expression tree ending at \p V. If V is a store, the size is the width of
372 /// the stored value. Otherwise, the size is the width of the largest loaded
373 /// value reaching V. This method is used by the vectorizer to calculate
374 /// vectorization factors.
375 unsigned getVectorElementSize(Value *V);
377 /// Compute the minimum type sizes required to represent the entries in a
378 /// vectorizable tree.
379 void computeMinimumValueSizes();
381 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
382 unsigned getMaxVecRegSize() const {
383 return MaxVecRegSize;
386 // \returns minimum vector register size as set by cl::opt.
387 unsigned getMinVecRegSize() const {
388 return MinVecRegSize;
391 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
393 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
394 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
399 /// \returns the cost of the vectorizable entry.
400 int getEntryCost(TreeEntry *E);
402 /// This is the recursive part of buildTree.
403 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth);
405 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
406 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
407 bool canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const;
409 /// Vectorize a single entry in the tree.
410 Value *vectorizeTree(TreeEntry *E);
412 /// Vectorize a single entry in the tree, starting in \p VL.
413 Value *vectorizeTree(ArrayRef<Value *> VL);
415 /// \returns the pointer to the vectorized value if \p VL is already
416 /// vectorized, or NULL. They may happen in cycles.
417 Value *alreadyVectorized(ArrayRef<Value *> VL) const;
419 /// \returns the scalarization cost for this type. Scalarization in this
420 /// context means the creation of vectors from a group of scalars.
421 int getGatherCost(Type *Ty);
423 /// \returns the scalarization cost for this list of values. Assuming that
424 /// this subtree gets vectorized, we may need to extract the values from the
425 /// roots. This method calculates the cost of extracting the values.
426 int getGatherCost(ArrayRef<Value *> VL);
428 /// \brief Set the Builder insert point to one after the last instruction in
430 void setInsertPointAfterBundle(ArrayRef<Value *> VL);
432 /// \returns a vector from a collection of scalars in \p VL.
433 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
435 /// \returns whether the VectorizableTree is fully vectorizable and will
436 /// be beneficial even the tree height is tiny.
437 bool isFullyVectorizableTinyTree();
439 /// \reorder commutative operands in alt shuffle if they result in
441 void reorderAltShuffleOperands(ArrayRef<Value *> VL,
442 SmallVectorImpl<Value *> &Left,
443 SmallVectorImpl<Value *> &Right);
444 /// \reorder commutative operands to get better probability of
445 /// generating vectorized code.
446 void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
447 SmallVectorImpl<Value *> &Left,
448 SmallVectorImpl<Value *> &Right);
450 TreeEntry() : Scalars(), VectorizedValue(nullptr),
453 /// \returns true if the scalars in VL are equal to this entry.
454 bool isSame(ArrayRef<Value *> VL) const {
455 assert(VL.size() == Scalars.size() && "Invalid size");
456 return std::equal(VL.begin(), VL.end(), Scalars.begin());
459 /// A vector of scalars.
462 /// The Scalars are vectorized into this value. It is initialized to Null.
463 Value *VectorizedValue;
465 /// Do we need to gather this sequence ?
469 /// Create a new VectorizableTree entry.
470 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized) {
471 VectorizableTree.emplace_back();
472 int idx = VectorizableTree.size() - 1;
473 TreeEntry *Last = &VectorizableTree[idx];
474 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
475 Last->NeedToGather = !Vectorized;
477 for (int i = 0, e = VL.size(); i != e; ++i) {
478 assert(!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!");
479 ScalarToTreeEntry[VL[i]] = idx;
482 MustGather.insert(VL.begin(), VL.end());
487 /// -- Vectorization State --
488 /// Holds all of the tree entries.
489 std::vector<TreeEntry> VectorizableTree;
491 /// Maps a specific scalar to its tree entry.
492 SmallDenseMap<Value*, int> ScalarToTreeEntry;
494 /// A list of scalars that we found that we need to keep as scalars.
497 /// This POD struct describes one external user in the vectorized tree.
498 struct ExternalUser {
499 ExternalUser (Value *S, llvm::User *U, int L) :
500 Scalar(S), User(U), Lane(L){}
501 // Which scalar in our function.
503 // Which user that uses the scalar.
505 // Which lane does the scalar belong to.
508 typedef SmallVector<ExternalUser, 16> UserList;
510 /// Checks if two instructions may access the same memory.
512 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
513 /// is invariant in the calling loop.
514 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
515 Instruction *Inst2) {
517 // First check if the result is already in the cache.
518 AliasCacheKey key = std::make_pair(Inst1, Inst2);
519 Optional<bool> &result = AliasCache[key];
520 if (result.hasValue()) {
521 return result.getValue();
523 MemoryLocation Loc2 = getLocation(Inst2, AA);
525 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
526 // Do the alias check.
527 aliased = AA->alias(Loc1, Loc2);
529 // Store the result in the cache.
534 typedef std::pair<Instruction *, Instruction *> AliasCacheKey;
536 /// Cache for alias results.
537 /// TODO: consider moving this to the AliasAnalysis itself.
538 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
540 /// Removes an instruction from its block and eventually deletes it.
541 /// It's like Instruction::eraseFromParent() except that the actual deletion
542 /// is delayed until BoUpSLP is destructed.
543 /// This is required to ensure that there are no incorrect collisions in the
544 /// AliasCache, which can happen if a new instruction is allocated at the
545 /// same address as a previously deleted instruction.
546 void eraseInstruction(Instruction *I) {
547 I->removeFromParent();
548 I->dropAllReferences();
549 DeletedInstructions.push_back(std::unique_ptr<Instruction>(I));
552 /// Temporary store for deleted instructions. Instructions will be deleted
553 /// eventually when the BoUpSLP is destructed.
554 SmallVector<std::unique_ptr<Instruction>, 8> DeletedInstructions;
556 /// A list of values that need to extracted out of the tree.
557 /// This list holds pairs of (Internal Scalar : External User).
558 UserList ExternalUses;
560 /// Values used only by @llvm.assume calls.
561 SmallPtrSet<const Value *, 32> EphValues;
563 /// Holds all of the instructions that we gathered.
564 SetVector<Instruction *> GatherSeq;
565 /// A list of blocks that we are going to CSE.
566 SetVector<BasicBlock *> CSEBlocks;
568 /// Contains all scheduling relevant data for an instruction.
569 /// A ScheduleData either represents a single instruction or a member of an
570 /// instruction bundle (= a group of instructions which is combined into a
571 /// vector instruction).
572 struct ScheduleData {
574 // The initial value for the dependency counters. It means that the
575 // dependencies are not calculated yet.
576 enum { InvalidDeps = -1 };
579 : Inst(nullptr), FirstInBundle(nullptr), NextInBundle(nullptr),
580 NextLoadStore(nullptr), SchedulingRegionID(0), SchedulingPriority(0),
581 Dependencies(InvalidDeps), UnscheduledDeps(InvalidDeps),
582 UnscheduledDepsInBundle(InvalidDeps), IsScheduled(false) {}
584 void init(int BlockSchedulingRegionID) {
585 FirstInBundle = this;
586 NextInBundle = nullptr;
587 NextLoadStore = nullptr;
589 SchedulingRegionID = BlockSchedulingRegionID;
590 UnscheduledDepsInBundle = UnscheduledDeps;
594 /// Returns true if the dependency information has been calculated.
595 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
597 /// Returns true for single instructions and for bundle representatives
598 /// (= the head of a bundle).
599 bool isSchedulingEntity() const { return FirstInBundle == this; }
601 /// Returns true if it represents an instruction bundle and not only a
602 /// single instruction.
603 bool isPartOfBundle() const {
604 return NextInBundle != nullptr || FirstInBundle != this;
607 /// Returns true if it is ready for scheduling, i.e. it has no more
608 /// unscheduled depending instructions/bundles.
609 bool isReady() const {
610 assert(isSchedulingEntity() &&
611 "can't consider non-scheduling entity for ready list");
612 return UnscheduledDepsInBundle == 0 && !IsScheduled;
615 /// Modifies the number of unscheduled dependencies, also updating it for
616 /// the whole bundle.
617 int incrementUnscheduledDeps(int Incr) {
618 UnscheduledDeps += Incr;
619 return FirstInBundle->UnscheduledDepsInBundle += Incr;
622 /// Sets the number of unscheduled dependencies to the number of
624 void resetUnscheduledDeps() {
625 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
628 /// Clears all dependency information.
629 void clearDependencies() {
630 Dependencies = InvalidDeps;
631 resetUnscheduledDeps();
632 MemoryDependencies.clear();
635 void dump(raw_ostream &os) const {
636 if (!isSchedulingEntity()) {
638 } else if (NextInBundle) {
640 ScheduleData *SD = NextInBundle;
642 os << ';' << *SD->Inst;
643 SD = SD->NextInBundle;
653 /// Points to the head in an instruction bundle (and always to this for
654 /// single instructions).
655 ScheduleData *FirstInBundle;
657 /// Single linked list of all instructions in a bundle. Null if it is a
658 /// single instruction.
659 ScheduleData *NextInBundle;
661 /// Single linked list of all memory instructions (e.g. load, store, call)
662 /// in the block - until the end of the scheduling region.
663 ScheduleData *NextLoadStore;
665 /// The dependent memory instructions.
666 /// This list is derived on demand in calculateDependencies().
667 SmallVector<ScheduleData *, 4> MemoryDependencies;
669 /// This ScheduleData is in the current scheduling region if this matches
670 /// the current SchedulingRegionID of BlockScheduling.
671 int SchedulingRegionID;
673 /// Used for getting a "good" final ordering of instructions.
674 int SchedulingPriority;
676 /// The number of dependencies. Constitutes of the number of users of the
677 /// instruction plus the number of dependent memory instructions (if any).
678 /// This value is calculated on demand.
679 /// If InvalidDeps, the number of dependencies is not calculated yet.
683 /// The number of dependencies minus the number of dependencies of scheduled
684 /// instructions. As soon as this is zero, the instruction/bundle gets ready
686 /// Note that this is negative as long as Dependencies is not calculated.
689 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
690 /// single instructions.
691 int UnscheduledDepsInBundle;
693 /// True if this instruction is scheduled (or considered as scheduled in the
699 friend inline raw_ostream &operator<<(raw_ostream &os,
700 const BoUpSLP::ScheduleData &SD) {
706 /// Contains all scheduling data for a basic block.
708 struct BlockScheduling {
710 BlockScheduling(BasicBlock *BB)
711 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize),
712 ScheduleStart(nullptr), ScheduleEnd(nullptr),
713 FirstLoadStoreInRegion(nullptr), LastLoadStoreInRegion(nullptr),
714 ScheduleRegionSize(0),
715 ScheduleRegionSizeLimit(ScheduleRegionSizeBudget),
716 // Make sure that the initial SchedulingRegionID is greater than the
717 // initial SchedulingRegionID in ScheduleData (which is 0).
718 SchedulingRegionID(1) {}
722 ScheduleStart = nullptr;
723 ScheduleEnd = nullptr;
724 FirstLoadStoreInRegion = nullptr;
725 LastLoadStoreInRegion = nullptr;
727 // Reduce the maximum schedule region size by the size of the
728 // previous scheduling run.
729 ScheduleRegionSizeLimit -= ScheduleRegionSize;
730 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
731 ScheduleRegionSizeLimit = MinScheduleRegionSize;
732 ScheduleRegionSize = 0;
734 // Make a new scheduling region, i.e. all existing ScheduleData is not
735 // in the new region yet.
736 ++SchedulingRegionID;
739 ScheduleData *getScheduleData(Value *V) {
740 ScheduleData *SD = ScheduleDataMap[V];
741 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
746 bool isInSchedulingRegion(ScheduleData *SD) {
747 return SD->SchedulingRegionID == SchedulingRegionID;
750 /// Marks an instruction as scheduled and puts all dependent ready
751 /// instructions into the ready-list.
752 template <typename ReadyListType>
753 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
754 SD->IsScheduled = true;
755 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
757 ScheduleData *BundleMember = SD;
758 while (BundleMember) {
759 // Handle the def-use chain dependencies.
760 for (Use &U : BundleMember->Inst->operands()) {
761 ScheduleData *OpDef = getScheduleData(U.get());
762 if (OpDef && OpDef->hasValidDependencies() &&
763 OpDef->incrementUnscheduledDeps(-1) == 0) {
764 // There are no more unscheduled dependencies after decrementing,
765 // so we can put the dependent instruction into the ready list.
766 ScheduleData *DepBundle = OpDef->FirstInBundle;
767 assert(!DepBundle->IsScheduled &&
768 "already scheduled bundle gets ready");
769 ReadyList.insert(DepBundle);
770 DEBUG(dbgs() << "SLP: gets ready (def): " << *DepBundle << "\n");
773 // Handle the memory dependencies.
774 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
775 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
776 // There are no more unscheduled dependencies after decrementing,
777 // so we can put the dependent instruction into the ready list.
778 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
779 assert(!DepBundle->IsScheduled &&
780 "already scheduled bundle gets ready");
781 ReadyList.insert(DepBundle);
782 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle << "\n");
785 BundleMember = BundleMember->NextInBundle;
789 /// Put all instructions into the ReadyList which are ready for scheduling.
790 template <typename ReadyListType>
791 void initialFillReadyList(ReadyListType &ReadyList) {
792 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
793 ScheduleData *SD = getScheduleData(I);
794 if (SD->isSchedulingEntity() && SD->isReady()) {
795 ReadyList.insert(SD);
796 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n");
801 /// Checks if a bundle of instructions can be scheduled, i.e. has no
802 /// cyclic dependencies. This is only a dry-run, no instructions are
803 /// actually moved at this stage.
804 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP);
806 /// Un-bundles a group of instructions.
807 void cancelScheduling(ArrayRef<Value *> VL);
809 /// Extends the scheduling region so that V is inside the region.
810 /// \returns true if the region size is within the limit.
811 bool extendSchedulingRegion(Value *V);
813 /// Initialize the ScheduleData structures for new instructions in the
814 /// scheduling region.
815 void initScheduleData(Instruction *FromI, Instruction *ToI,
816 ScheduleData *PrevLoadStore,
817 ScheduleData *NextLoadStore);
819 /// Updates the dependency information of a bundle and of all instructions/
820 /// bundles which depend on the original bundle.
821 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
824 /// Sets all instruction in the scheduling region to un-scheduled.
825 void resetSchedule();
829 /// Simple memory allocation for ScheduleData.
830 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
832 /// The size of a ScheduleData array in ScheduleDataChunks.
835 /// The allocator position in the current chunk, which is the last entry
836 /// of ScheduleDataChunks.
839 /// Attaches ScheduleData to Instruction.
840 /// Note that the mapping survives during all vectorization iterations, i.e.
841 /// ScheduleData structures are recycled.
842 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
844 struct ReadyList : SmallVector<ScheduleData *, 8> {
845 void insert(ScheduleData *SD) { push_back(SD); }
848 /// The ready-list for scheduling (only used for the dry-run).
849 ReadyList ReadyInsts;
851 /// The first instruction of the scheduling region.
852 Instruction *ScheduleStart;
854 /// The first instruction _after_ the scheduling region.
855 Instruction *ScheduleEnd;
857 /// The first memory accessing instruction in the scheduling region
859 ScheduleData *FirstLoadStoreInRegion;
861 /// The last memory accessing instruction in the scheduling region
863 ScheduleData *LastLoadStoreInRegion;
865 /// The current size of the scheduling region.
866 int ScheduleRegionSize;
868 /// The maximum size allowed for the scheduling region.
869 int ScheduleRegionSizeLimit;
871 /// The ID of the scheduling region. For a new vectorization iteration this
872 /// is incremented which "removes" all ScheduleData from the region.
873 int SchedulingRegionID;
876 /// Attaches the BlockScheduling structures to basic blocks.
877 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
879 /// Performs the "real" scheduling. Done before vectorization is actually
880 /// performed in a basic block.
881 void scheduleBlock(BlockScheduling *BS);
883 /// List of users to ignore during scheduling and that don't need extracting.
884 ArrayRef<Value *> UserIgnoreList;
886 // Number of load-bundles, which contain consecutive loads.
887 int NumLoadsWantToKeepOrder;
889 // Number of load-bundles of size 2, which are consecutive loads if reversed.
890 int NumLoadsWantToChangeOrder;
892 // Analysis and block reference.
895 TargetTransformInfo *TTI;
896 TargetLibraryInfo *TLI;
902 const DataLayout *DL;
903 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
904 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
905 /// Instruction builder to construct the vectorized tree.
908 /// A map of scalar integer values to the smallest bit width with which they
909 /// can legally be represented.
910 MapVector<Value *, uint64_t> MinBWs;
913 } // end namespace llvm
914 } // end namespace slpvectorizer
916 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
917 ArrayRef<Value *> UserIgnoreLst) {
919 UserIgnoreList = UserIgnoreLst;
920 if (!getSameType(Roots))
922 buildTree_rec(Roots, 0);
924 // Collect the values that we need to extract from the tree.
925 for (TreeEntry &EIdx : VectorizableTree) {
926 TreeEntry *Entry = &EIdx;
929 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
930 Value *Scalar = Entry->Scalars[Lane];
932 // No need to handle users of gathered values.
933 if (Entry->NeedToGather)
936 for (User *U : Scalar->users()) {
937 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
939 Instruction *UserInst = dyn_cast<Instruction>(U);
943 // Skip in-tree scalars that become vectors
944 if (ScalarToTreeEntry.count(U)) {
945 int Idx = ScalarToTreeEntry[U];
946 TreeEntry *UseEntry = &VectorizableTree[Idx];
947 Value *UseScalar = UseEntry->Scalars[0];
948 // Some in-tree scalars will remain as scalar in vectorized
949 // instructions. If that is the case, the one in Lane 0 will
951 if (UseScalar != U ||
952 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
953 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
955 assert(!VectorizableTree[Idx].NeedToGather && "Bad state");
960 // Ignore users in the user ignore list.
961 if (std::find(UserIgnoreList.begin(), UserIgnoreList.end(), UserInst) !=
962 UserIgnoreList.end())
965 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<
966 Lane << " from " << *Scalar << ".\n");
967 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
974 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth) {
975 bool SameTy = allConstant(VL) || getSameType(VL); (void)SameTy;
976 bool isAltShuffle = false;
977 assert(SameTy && "Invalid types!");
979 if (Depth == RecursionMaxDepth) {
980 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
981 newTreeEntry(VL, false);
985 // Don't handle vectors.
986 if (VL[0]->getType()->isVectorTy()) {
987 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
988 newTreeEntry(VL, false);
992 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
993 if (SI->getValueOperand()->getType()->isVectorTy()) {
994 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
995 newTreeEntry(VL, false);
998 unsigned Opcode = getSameOpcode(VL);
1000 // Check that this shuffle vector refers to the alternate
1001 // sequence of opcodes.
1002 if (Opcode == Instruction::ShuffleVector) {
1003 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
1004 unsigned Op = I0->getOpcode();
1005 if (Op != Instruction::ShuffleVector)
1006 isAltShuffle = true;
1009 // If all of the operands are identical or constant we have a simple solution.
1010 if (allConstant(VL) || isSplat(VL) || !getSameBlock(VL) || !Opcode) {
1011 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1012 newTreeEntry(VL, false);
1016 // We now know that this is a vector of instructions of the same type from
1019 // Don't vectorize ephemeral values.
1020 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1021 if (EphValues.count(VL[i])) {
1022 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1023 ") is ephemeral.\n");
1024 newTreeEntry(VL, false);
1029 // Check if this is a duplicate of another entry.
1030 if (ScalarToTreeEntry.count(VL[0])) {
1031 int Idx = ScalarToTreeEntry[VL[0]];
1032 TreeEntry *E = &VectorizableTree[Idx];
1033 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1034 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n");
1035 if (E->Scalars[i] != VL[i]) {
1036 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1037 newTreeEntry(VL, false);
1041 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *VL[0] << ".\n");
1045 // Check that none of the instructions in the bundle are already in the tree.
1046 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1047 if (ScalarToTreeEntry.count(VL[i])) {
1048 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1049 ") is already in tree.\n");
1050 newTreeEntry(VL, false);
1055 // If any of the scalars is marked as a value that needs to stay scalar then
1056 // we need to gather the scalars.
1057 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1058 if (MustGather.count(VL[i])) {
1059 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1060 newTreeEntry(VL, false);
1065 // Check that all of the users of the scalars that we want to vectorize are
1067 Instruction *VL0 = cast<Instruction>(VL[0]);
1068 BasicBlock *BB = cast<Instruction>(VL0)->getParent();
1070 if (!DT->isReachableFromEntry(BB)) {
1071 // Don't go into unreachable blocks. They may contain instructions with
1072 // dependency cycles which confuse the final scheduling.
1073 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1074 newTreeEntry(VL, false);
1078 // Check that every instructions appears once in this bundle.
1079 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1080 for (unsigned j = i+1; j < e; ++j)
1081 if (VL[i] == VL[j]) {
1082 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1083 newTreeEntry(VL, false);
1087 auto &BSRef = BlocksSchedules[BB];
1089 BSRef = llvm::make_unique<BlockScheduling>(BB);
1091 BlockScheduling &BS = *BSRef.get();
1093 if (!BS.tryScheduleBundle(VL, this)) {
1094 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1095 assert((!BS.getScheduleData(VL[0]) ||
1096 !BS.getScheduleData(VL[0])->isPartOfBundle()) &&
1097 "tryScheduleBundle should cancelScheduling on failure");
1098 newTreeEntry(VL, false);
1101 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1104 case Instruction::PHI: {
1105 PHINode *PH = dyn_cast<PHINode>(VL0);
1107 // Check for terminator values (e.g. invoke).
1108 for (unsigned j = 0; j < VL.size(); ++j)
1109 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1110 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1111 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1113 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n");
1114 BS.cancelScheduling(VL);
1115 newTreeEntry(VL, false);
1120 newTreeEntry(VL, true);
1121 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1123 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1125 // Prepare the operand vector.
1127 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1128 PH->getIncomingBlock(i)));
1130 buildTree_rec(Operands, Depth + 1);
1134 case Instruction::ExtractValue:
1135 case Instruction::ExtractElement: {
1136 bool Reuse = canReuseExtract(VL, Opcode);
1138 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n");
1140 BS.cancelScheduling(VL);
1142 newTreeEntry(VL, Reuse);
1145 case Instruction::Load: {
1146 // Check that a vectorized load would load the same memory as a scalar
1148 // For example we don't want vectorize loads that are smaller than 8 bit.
1149 // Even though we have a packed struct {<i2, i2, i2, i2>} LLVM treats
1150 // loading/storing it as an i8 struct. If we vectorize loads/stores from
1151 // such a struct we read/write packed bits disagreeing with the
1152 // unvectorized version.
1153 Type *ScalarTy = VL[0]->getType();
1155 if (DL->getTypeSizeInBits(ScalarTy) !=
1156 DL->getTypeAllocSizeInBits(ScalarTy)) {
1157 BS.cancelScheduling(VL);
1158 newTreeEntry(VL, false);
1159 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1162 // Check if the loads are consecutive or of we need to swizzle them.
1163 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1164 LoadInst *L = cast<LoadInst>(VL[i]);
1165 if (!L->isSimple()) {
1166 BS.cancelScheduling(VL);
1167 newTreeEntry(VL, false);
1168 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1172 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1173 if (VL.size() == 2 && isConsecutiveAccess(VL[1], VL[0], *DL, *SE)) {
1174 ++NumLoadsWantToChangeOrder;
1176 BS.cancelScheduling(VL);
1177 newTreeEntry(VL, false);
1178 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1182 ++NumLoadsWantToKeepOrder;
1183 newTreeEntry(VL, true);
1184 DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1187 case Instruction::ZExt:
1188 case Instruction::SExt:
1189 case Instruction::FPToUI:
1190 case Instruction::FPToSI:
1191 case Instruction::FPExt:
1192 case Instruction::PtrToInt:
1193 case Instruction::IntToPtr:
1194 case Instruction::SIToFP:
1195 case Instruction::UIToFP:
1196 case Instruction::Trunc:
1197 case Instruction::FPTrunc:
1198 case Instruction::BitCast: {
1199 Type *SrcTy = VL0->getOperand(0)->getType();
1200 for (unsigned i = 0; i < VL.size(); ++i) {
1201 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1202 if (Ty != SrcTy || !isValidElementType(Ty)) {
1203 BS.cancelScheduling(VL);
1204 newTreeEntry(VL, false);
1205 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n");
1209 newTreeEntry(VL, true);
1210 DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1212 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1214 // Prepare the operand vector.
1216 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1218 buildTree_rec(Operands, Depth+1);
1222 case Instruction::ICmp:
1223 case Instruction::FCmp: {
1224 // Check that all of the compares have the same predicate.
1225 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1226 Type *ComparedTy = cast<Instruction>(VL[0])->getOperand(0)->getType();
1227 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1228 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1229 if (Cmp->getPredicate() != P0 ||
1230 Cmp->getOperand(0)->getType() != ComparedTy) {
1231 BS.cancelScheduling(VL);
1232 newTreeEntry(VL, false);
1233 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
1238 newTreeEntry(VL, true);
1239 DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1241 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1243 // Prepare the operand vector.
1245 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1247 buildTree_rec(Operands, Depth+1);
1251 case Instruction::Select:
1252 case Instruction::Add:
1253 case Instruction::FAdd:
1254 case Instruction::Sub:
1255 case Instruction::FSub:
1256 case Instruction::Mul:
1257 case Instruction::FMul:
1258 case Instruction::UDiv:
1259 case Instruction::SDiv:
1260 case Instruction::FDiv:
1261 case Instruction::URem:
1262 case Instruction::SRem:
1263 case Instruction::FRem:
1264 case Instruction::Shl:
1265 case Instruction::LShr:
1266 case Instruction::AShr:
1267 case Instruction::And:
1268 case Instruction::Or:
1269 case Instruction::Xor: {
1270 newTreeEntry(VL, true);
1271 DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1273 // Sort operands of the instructions so that each side is more likely to
1274 // have the same opcode.
1275 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1276 ValueList Left, Right;
1277 reorderInputsAccordingToOpcode(VL, Left, Right);
1278 buildTree_rec(Left, Depth + 1);
1279 buildTree_rec(Right, Depth + 1);
1283 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1285 // Prepare the operand vector.
1287 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1289 buildTree_rec(Operands, Depth+1);
1293 case Instruction::GetElementPtr: {
1294 // We don't combine GEPs with complicated (nested) indexing.
1295 for (unsigned j = 0; j < VL.size(); ++j) {
1296 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1297 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1298 BS.cancelScheduling(VL);
1299 newTreeEntry(VL, false);
1304 // We can't combine several GEPs into one vector if they operate on
1306 Type *Ty0 = cast<Instruction>(VL0)->getOperand(0)->getType();
1307 for (unsigned j = 0; j < VL.size(); ++j) {
1308 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1310 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
1311 BS.cancelScheduling(VL);
1312 newTreeEntry(VL, false);
1317 // We don't combine GEPs with non-constant indexes.
1318 for (unsigned j = 0; j < VL.size(); ++j) {
1319 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1320 if (!isa<ConstantInt>(Op)) {
1322 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1323 BS.cancelScheduling(VL);
1324 newTreeEntry(VL, false);
1329 newTreeEntry(VL, true);
1330 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1331 for (unsigned i = 0, e = 2; i < e; ++i) {
1333 // Prepare the operand vector.
1335 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1337 buildTree_rec(Operands, Depth + 1);
1341 case Instruction::Store: {
1342 // Check if the stores are consecutive or of we need to swizzle them.
1343 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1344 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1345 BS.cancelScheduling(VL);
1346 newTreeEntry(VL, false);
1347 DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1351 newTreeEntry(VL, true);
1352 DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1356 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1358 buildTree_rec(Operands, Depth + 1);
1361 case Instruction::Call: {
1362 // Check if the calls are all to the same vectorizable intrinsic.
1363 CallInst *CI = cast<CallInst>(VL[0]);
1364 // Check if this is an Intrinsic call or something that can be
1365 // represented by an intrinsic call
1366 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1367 if (!isTriviallyVectorizable(ID)) {
1368 BS.cancelScheduling(VL);
1369 newTreeEntry(VL, false);
1370 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1373 Function *Int = CI->getCalledFunction();
1374 Value *A1I = nullptr;
1375 if (hasVectorInstrinsicScalarOpd(ID, 1))
1376 A1I = CI->getArgOperand(1);
1377 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1378 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1379 if (!CI2 || CI2->getCalledFunction() != Int ||
1380 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1381 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1382 BS.cancelScheduling(VL);
1383 newTreeEntry(VL, false);
1384 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1388 // ctlz,cttz and powi are special intrinsics whose second argument
1389 // should be same in order for them to be vectorized.
1390 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1391 Value *A1J = CI2->getArgOperand(1);
1393 BS.cancelScheduling(VL);
1394 newTreeEntry(VL, false);
1395 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1396 << " argument "<< A1I<<"!=" << A1J
1401 // Verify that the bundle operands are identical between the two calls.
1402 if (CI->hasOperandBundles() &&
1403 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1404 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1405 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1406 BS.cancelScheduling(VL);
1407 newTreeEntry(VL, false);
1408 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="
1414 newTreeEntry(VL, true);
1415 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1417 // Prepare the operand vector.
1418 for (Value *j : VL) {
1419 CallInst *CI2 = dyn_cast<CallInst>(j);
1420 Operands.push_back(CI2->getArgOperand(i));
1422 buildTree_rec(Operands, Depth + 1);
1426 case Instruction::ShuffleVector: {
1427 // If this is not an alternate sequence of opcode like add-sub
1428 // then do not vectorize this instruction.
1429 if (!isAltShuffle) {
1430 BS.cancelScheduling(VL);
1431 newTreeEntry(VL, false);
1432 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1435 newTreeEntry(VL, true);
1436 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1438 // Reorder operands if reordering would enable vectorization.
1439 if (isa<BinaryOperator>(VL0)) {
1440 ValueList Left, Right;
1441 reorderAltShuffleOperands(VL, Left, Right);
1442 buildTree_rec(Left, Depth + 1);
1443 buildTree_rec(Right, Depth + 1);
1447 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1449 // Prepare the operand vector.
1451 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1453 buildTree_rec(Operands, Depth + 1);
1458 BS.cancelScheduling(VL);
1459 newTreeEntry(VL, false);
1460 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1465 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1468 auto *ST = dyn_cast<StructType>(T);
1470 N = ST->getNumElements();
1471 EltTy = *ST->element_begin();
1473 N = cast<ArrayType>(T)->getNumElements();
1474 EltTy = cast<ArrayType>(T)->getElementType();
1476 if (!isValidElementType(EltTy))
1478 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1479 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1482 // Check that struct is homogeneous.
1483 for (const auto *Ty : ST->elements())
1490 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const {
1491 assert(Opcode == Instruction::ExtractElement ||
1492 Opcode == Instruction::ExtractValue);
1493 assert(Opcode == getSameOpcode(VL) && "Invalid opcode");
1494 // Check if all of the extracts come from the same vector and from the
1497 Instruction *E0 = cast<Instruction>(VL0);
1498 Value *Vec = E0->getOperand(0);
1500 // We have to extract from a vector/aggregate with the same number of elements.
1502 if (Opcode == Instruction::ExtractValue) {
1503 const DataLayout &DL = E0->getModule()->getDataLayout();
1504 NElts = canMapToVector(Vec->getType(), DL);
1507 // Check if load can be rewritten as load of vector.
1508 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1509 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1512 NElts = Vec->getType()->getVectorNumElements();
1515 if (NElts != VL.size())
1518 // Check that all of the indices extract from the correct offset.
1519 if (!matchExtractIndex(E0, 0, Opcode))
1522 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1523 Instruction *E = cast<Instruction>(VL[i]);
1524 if (!matchExtractIndex(E, i, Opcode))
1526 if (E->getOperand(0) != Vec)
1533 int BoUpSLP::getEntryCost(TreeEntry *E) {
1534 ArrayRef<Value*> VL = E->Scalars;
1536 Type *ScalarTy = VL[0]->getType();
1537 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1538 ScalarTy = SI->getValueOperand()->getType();
1539 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1541 // If we have computed a smaller type for the expression, update VecTy so
1542 // that the costs will be accurate.
1543 if (MinBWs.count(VL[0]))
1544 VecTy = VectorType::get(IntegerType::get(F->getContext(), MinBWs[VL[0]]),
1547 if (E->NeedToGather) {
1548 if (allConstant(VL))
1551 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1553 return getGatherCost(E->Scalars);
1555 unsigned Opcode = getSameOpcode(VL);
1556 assert(Opcode && getSameType(VL) && getSameBlock(VL) && "Invalid VL");
1557 Instruction *VL0 = cast<Instruction>(VL[0]);
1559 case Instruction::PHI: {
1562 case Instruction::ExtractValue:
1563 case Instruction::ExtractElement: {
1564 if (canReuseExtract(VL, Opcode)) {
1566 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
1567 Instruction *E = cast<Instruction>(VL[i]);
1569 // Take credit for instruction that will become dead.
1571 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
1575 return getGatherCost(VecTy);
1577 case Instruction::ZExt:
1578 case Instruction::SExt:
1579 case Instruction::FPToUI:
1580 case Instruction::FPToSI:
1581 case Instruction::FPExt:
1582 case Instruction::PtrToInt:
1583 case Instruction::IntToPtr:
1584 case Instruction::SIToFP:
1585 case Instruction::UIToFP:
1586 case Instruction::Trunc:
1587 case Instruction::FPTrunc:
1588 case Instruction::BitCast: {
1589 Type *SrcTy = VL0->getOperand(0)->getType();
1591 // Calculate the cost of this instruction.
1592 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
1593 VL0->getType(), SrcTy);
1595 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
1596 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy);
1597 return VecCost - ScalarCost;
1599 case Instruction::FCmp:
1600 case Instruction::ICmp:
1601 case Instruction::Select: {
1602 // Calculate the cost of this instruction.
1603 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
1604 int ScalarCost = VecTy->getNumElements() *
1605 TTI->getCmpSelInstrCost(Opcode, ScalarTy, Builder.getInt1Ty());
1606 int VecCost = TTI->getCmpSelInstrCost(Opcode, VecTy, MaskTy);
1607 return VecCost - ScalarCost;
1609 case Instruction::Add:
1610 case Instruction::FAdd:
1611 case Instruction::Sub:
1612 case Instruction::FSub:
1613 case Instruction::Mul:
1614 case Instruction::FMul:
1615 case Instruction::UDiv:
1616 case Instruction::SDiv:
1617 case Instruction::FDiv:
1618 case Instruction::URem:
1619 case Instruction::SRem:
1620 case Instruction::FRem:
1621 case Instruction::Shl:
1622 case Instruction::LShr:
1623 case Instruction::AShr:
1624 case Instruction::And:
1625 case Instruction::Or:
1626 case Instruction::Xor: {
1627 // Certain instructions can be cheaper to vectorize if they have a
1628 // constant second vector operand.
1629 TargetTransformInfo::OperandValueKind Op1VK =
1630 TargetTransformInfo::OK_AnyValue;
1631 TargetTransformInfo::OperandValueKind Op2VK =
1632 TargetTransformInfo::OK_UniformConstantValue;
1633 TargetTransformInfo::OperandValueProperties Op1VP =
1634 TargetTransformInfo::OP_None;
1635 TargetTransformInfo::OperandValueProperties Op2VP =
1636 TargetTransformInfo::OP_None;
1638 // If all operands are exactly the same ConstantInt then set the
1639 // operand kind to OK_UniformConstantValue.
1640 // If instead not all operands are constants, then set the operand kind
1641 // to OK_AnyValue. If all operands are constants but not the same,
1642 // then set the operand kind to OK_NonUniformConstantValue.
1643 ConstantInt *CInt = nullptr;
1644 for (unsigned i = 0; i < VL.size(); ++i) {
1645 const Instruction *I = cast<Instruction>(VL[i]);
1646 if (!isa<ConstantInt>(I->getOperand(1))) {
1647 Op2VK = TargetTransformInfo::OK_AnyValue;
1651 CInt = cast<ConstantInt>(I->getOperand(1));
1654 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
1655 CInt != cast<ConstantInt>(I->getOperand(1)))
1656 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
1658 // FIXME: Currently cost of model modification for division by power of
1659 // 2 is handled for X86 and AArch64. Add support for other targets.
1660 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
1661 CInt->getValue().isPowerOf2())
1662 Op2VP = TargetTransformInfo::OP_PowerOf2;
1664 int ScalarCost = VecTy->getNumElements() *
1665 TTI->getArithmeticInstrCost(Opcode, ScalarTy, Op1VK,
1666 Op2VK, Op1VP, Op2VP);
1667 int VecCost = TTI->getArithmeticInstrCost(Opcode, VecTy, Op1VK, Op2VK,
1669 return VecCost - ScalarCost;
1671 case Instruction::GetElementPtr: {
1672 TargetTransformInfo::OperandValueKind Op1VK =
1673 TargetTransformInfo::OK_AnyValue;
1674 TargetTransformInfo::OperandValueKind Op2VK =
1675 TargetTransformInfo::OK_UniformConstantValue;
1678 VecTy->getNumElements() *
1679 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
1681 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
1683 return VecCost - ScalarCost;
1685 case Instruction::Load: {
1686 // Cost of wide load - cost of scalar loads.
1687 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
1688 int ScalarLdCost = VecTy->getNumElements() *
1689 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0);
1690 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
1691 VecTy, alignment, 0);
1692 return VecLdCost - ScalarLdCost;
1694 case Instruction::Store: {
1695 // We know that we can merge the stores. Calculate the cost.
1696 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
1697 int ScalarStCost = VecTy->getNumElements() *
1698 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0);
1699 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
1700 VecTy, alignment, 0);
1701 return VecStCost - ScalarStCost;
1703 case Instruction::Call: {
1704 CallInst *CI = cast<CallInst>(VL0);
1705 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1707 // Calculate the cost of the scalar and vector calls.
1708 SmallVector<Type*, 4> ScalarTys, VecTys;
1709 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op) {
1710 ScalarTys.push_back(CI->getArgOperand(op)->getType());
1711 VecTys.push_back(VectorType::get(CI->getArgOperand(op)->getType(),
1712 VecTy->getNumElements()));
1716 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
1717 FMF = FPMO->getFastMathFlags();
1719 int ScalarCallCost = VecTy->getNumElements() *
1720 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
1722 int VecCallCost = TTI->getIntrinsicInstrCost(ID, VecTy, VecTys, FMF);
1724 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCost
1725 << " (" << VecCallCost << "-" << ScalarCallCost << ")"
1726 << " for " << *CI << "\n");
1728 return VecCallCost - ScalarCallCost;
1730 case Instruction::ShuffleVector: {
1731 TargetTransformInfo::OperandValueKind Op1VK =
1732 TargetTransformInfo::OK_AnyValue;
1733 TargetTransformInfo::OperandValueKind Op2VK =
1734 TargetTransformInfo::OK_AnyValue;
1737 for (Value *i : VL) {
1738 Instruction *I = cast<Instruction>(i);
1742 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
1744 // VecCost is equal to sum of the cost of creating 2 vectors
1745 // and the cost of creating shuffle.
1746 Instruction *I0 = cast<Instruction>(VL[0]);
1748 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
1749 Instruction *I1 = cast<Instruction>(VL[1]);
1751 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
1753 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
1754 return VecCost - ScalarCost;
1757 llvm_unreachable("Unknown instruction");
1761 bool BoUpSLP::isFullyVectorizableTinyTree() {
1762 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<
1763 VectorizableTree.size() << " is fully vectorizable .\n");
1765 // We only handle trees of height 2.
1766 if (VectorizableTree.size() != 2)
1769 // Handle splat and all-constants stores.
1770 if (!VectorizableTree[0].NeedToGather &&
1771 (allConstant(VectorizableTree[1].Scalars) ||
1772 isSplat(VectorizableTree[1].Scalars)))
1775 // Gathering cost would be too much for tiny trees.
1776 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
1782 int BoUpSLP::getSpillCost() {
1783 // Walk from the bottom of the tree to the top, tracking which values are
1784 // live. When we see a call instruction that is not part of our tree,
1785 // query TTI to see if there is a cost to keeping values live over it
1786 // (for example, if spills and fills are required).
1787 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
1790 SmallPtrSet<Instruction*, 4> LiveValues;
1791 Instruction *PrevInst = nullptr;
1793 for (const auto &N : VectorizableTree) {
1794 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
1803 // Update LiveValues.
1804 LiveValues.erase(PrevInst);
1805 for (auto &J : PrevInst->operands()) {
1806 if (isa<Instruction>(&*J) && ScalarToTreeEntry.count(&*J))
1807 LiveValues.insert(cast<Instruction>(&*J));
1811 dbgs() << "SLP: #LV: " << LiveValues.size();
1812 for (auto *X : LiveValues)
1813 dbgs() << " " << X->getName();
1814 dbgs() << ", Looking at ";
1818 // Now find the sequence of instructions between PrevInst and Inst.
1819 BasicBlock::reverse_iterator InstIt(Inst->getIterator()),
1820 PrevInstIt(PrevInst->getIterator());
1822 while (InstIt != PrevInstIt) {
1823 if (PrevInstIt == PrevInst->getParent()->rend()) {
1824 PrevInstIt = Inst->getParent()->rbegin();
1828 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
1829 SmallVector<Type*, 4> V;
1830 for (auto *II : LiveValues)
1831 V.push_back(VectorType::get(II->getType(), BundleWidth));
1832 Cost += TTI->getCostOfKeepingLiveOverCall(V);
1844 int BoUpSLP::getTreeCost() {
1846 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<
1847 VectorizableTree.size() << ".\n");
1849 // We only vectorize tiny trees if it is fully vectorizable.
1850 if (VectorizableTree.size() < MinTreeSize && !isFullyVectorizableTinyTree()) {
1851 if (VectorizableTree.empty()) {
1852 assert(!ExternalUses.size() && "We should not have any external users");
1857 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
1859 for (TreeEntry &TE : VectorizableTree) {
1860 int C = getEntryCost(&TE);
1861 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "
1862 << *TE.Scalars[0] << ".\n");
1866 SmallSet<Value *, 16> ExtractCostCalculated;
1867 int ExtractCost = 0;
1868 for (ExternalUser &EU : ExternalUses) {
1869 // We only add extract cost once for the same scalar.
1870 if (!ExtractCostCalculated.insert(EU.Scalar).second)
1873 // Uses by ephemeral values are free (because the ephemeral value will be
1874 // removed prior to code generation, and so the extraction will be
1875 // removed as well).
1876 if (EphValues.count(EU.User))
1879 // If we plan to rewrite the tree in a smaller type, we will need to sign
1880 // extend the extracted value back to the original type. Here, we account
1881 // for the extract and the added cost of the sign extend if needed.
1882 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
1883 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
1884 if (MinBWs.count(ScalarRoot)) {
1885 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot]);
1886 VecTy = VectorType::get(MinTy, BundleWidth);
1887 ExtractCost += TTI->getExtractWithExtendCost(
1888 Instruction::SExt, EU.Scalar->getType(), VecTy, EU.Lane);
1891 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
1895 int SpillCost = getSpillCost();
1896 Cost += SpillCost + ExtractCost;
1898 DEBUG(dbgs() << "SLP: Spill Cost = " << SpillCost << ".\n"
1899 << "SLP: Extract Cost = " << ExtractCost << ".\n"
1900 << "SLP: Total Cost = " << Cost << ".\n");
1904 int BoUpSLP::getGatherCost(Type *Ty) {
1906 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
1907 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
1911 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
1912 // Find the type of the operands in VL.
1913 Type *ScalarTy = VL[0]->getType();
1914 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1915 ScalarTy = SI->getValueOperand()->getType();
1916 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1917 // Find the cost of inserting/extracting values from the vector.
1918 return getGatherCost(VecTy);
1921 // Reorder commutative operations in alternate shuffle if the resulting vectors
1922 // are consecutive loads. This would allow us to vectorize the tree.
1923 // If we have something like-
1924 // load a[0] - load b[0]
1925 // load b[1] + load a[1]
1926 // load a[2] - load b[2]
1927 // load a[3] + load b[3]
1928 // Reordering the second load b[1] load a[1] would allow us to vectorize this
1930 void BoUpSLP::reorderAltShuffleOperands(ArrayRef<Value *> VL,
1931 SmallVectorImpl<Value *> &Left,
1932 SmallVectorImpl<Value *> &Right) {
1933 // Push left and right operands of binary operation into Left and Right
1934 for (Value *i : VL) {
1935 Left.push_back(cast<Instruction>(i)->getOperand(0));
1936 Right.push_back(cast<Instruction>(i)->getOperand(1));
1939 // Reorder if we have a commutative operation and consecutive access
1940 // are on either side of the alternate instructions.
1941 for (unsigned j = 0; j < VL.size() - 1; ++j) {
1942 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
1943 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
1944 Instruction *VL1 = cast<Instruction>(VL[j]);
1945 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
1946 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
1947 std::swap(Left[j], Right[j]);
1949 } else if (VL2->isCommutative() &&
1950 isConsecutiveAccess(L, L1, *DL, *SE)) {
1951 std::swap(Left[j + 1], Right[j + 1]);
1957 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
1958 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
1959 Instruction *VL1 = cast<Instruction>(VL[j]);
1960 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
1961 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
1962 std::swap(Left[j], Right[j]);
1964 } else if (VL2->isCommutative() &&
1965 isConsecutiveAccess(L, L1, *DL, *SE)) {
1966 std::swap(Left[j + 1], Right[j + 1]);
1975 // Return true if I should be commuted before adding it's left and right
1976 // operands to the arrays Left and Right.
1978 // The vectorizer is trying to either have all elements one side being
1979 // instruction with the same opcode to enable further vectorization, or having
1980 // a splat to lower the vectorizing cost.
1981 static bool shouldReorderOperands(int i, Instruction &I,
1982 SmallVectorImpl<Value *> &Left,
1983 SmallVectorImpl<Value *> &Right,
1984 bool AllSameOpcodeLeft,
1985 bool AllSameOpcodeRight, bool SplatLeft,
1987 Value *VLeft = I.getOperand(0);
1988 Value *VRight = I.getOperand(1);
1989 // If we have "SplatRight", try to see if commuting is needed to preserve it.
1991 if (VRight == Right[i - 1])
1992 // Preserve SplatRight
1994 if (VLeft == Right[i - 1]) {
1995 // Commuting would preserve SplatRight, but we don't want to break
1996 // SplatLeft either, i.e. preserve the original order if possible.
1997 // (FIXME: why do we care?)
1998 if (SplatLeft && VLeft == Left[i - 1])
2003 // Symmetrically handle Right side.
2005 if (VLeft == Left[i - 1])
2006 // Preserve SplatLeft
2008 if (VRight == Left[i - 1])
2012 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2013 Instruction *IRight = dyn_cast<Instruction>(VRight);
2015 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2016 // it and not the right, in this case we want to commute.
2017 if (AllSameOpcodeRight) {
2018 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2019 if (IRight && RightPrevOpcode == IRight->getOpcode())
2020 // Do not commute, a match on the right preserves AllSameOpcodeRight
2022 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2023 // We have a match and may want to commute, but first check if there is
2024 // not also a match on the existing operands on the Left to preserve
2025 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2026 // (FIXME: why do we care?)
2027 if (AllSameOpcodeLeft && ILeft &&
2028 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2033 // Symmetrically handle Left side.
2034 if (AllSameOpcodeLeft) {
2035 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2036 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2038 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2044 void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
2045 SmallVectorImpl<Value *> &Left,
2046 SmallVectorImpl<Value *> &Right) {
2049 // Peel the first iteration out of the loop since there's nothing
2050 // interesting to do anyway and it simplifies the checks in the loop.
2051 auto VLeft = cast<Instruction>(VL[0])->getOperand(0);
2052 auto VRight = cast<Instruction>(VL[0])->getOperand(1);
2053 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2054 // Favor having instruction to the right. FIXME: why?
2055 std::swap(VLeft, VRight);
2056 Left.push_back(VLeft);
2057 Right.push_back(VRight);
2060 // Keep track if we have instructions with all the same opcode on one side.
2061 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2062 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2063 // Keep track if we have one side with all the same value (broadcast).
2064 bool SplatLeft = true;
2065 bool SplatRight = true;
2067 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2068 Instruction *I = cast<Instruction>(VL[i]);
2069 assert(I->isCommutative() && "Can only process commutative instruction");
2070 // Commute to favor either a splat or maximizing having the same opcodes on
2072 if (shouldReorderOperands(i, *I, Left, Right, AllSameOpcodeLeft,
2073 AllSameOpcodeRight, SplatLeft, SplatRight)) {
2074 Left.push_back(I->getOperand(1));
2075 Right.push_back(I->getOperand(0));
2077 Left.push_back(I->getOperand(0));
2078 Right.push_back(I->getOperand(1));
2080 // Update Splat* and AllSameOpcode* after the insertion.
2081 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2082 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2083 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2084 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2085 cast<Instruction>(Left[i])->getOpcode());
2086 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2087 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2088 cast<Instruction>(Right[i])->getOpcode());
2091 // If one operand end up being broadcast, return this operand order.
2092 if (SplatRight || SplatLeft)
2095 // Finally check if we can get longer vectorizable chain by reordering
2096 // without breaking the good operand order detected above.
2097 // E.g. If we have something like-
2098 // load a[0] load b[0]
2099 // load b[1] load a[1]
2100 // load a[2] load b[2]
2101 // load a[3] load b[3]
2102 // Reordering the second load b[1] load a[1] would allow us to vectorize
2103 // this code and we still retain AllSameOpcode property.
2104 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2106 // add a[0],c[0] load b[0]
2107 // add a[1],c[2] load b[1]
2109 // add a[3],c[3] load b[3]
2110 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2111 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2112 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2113 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2114 std::swap(Left[j + 1], Right[j + 1]);
2119 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2120 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2121 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2122 std::swap(Left[j + 1], Right[j + 1]);
2131 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL) {
2133 // Get the basic block this bundle is in. All instructions in the bundle
2134 // should be in this block.
2135 auto *Front = cast<Instruction>(VL.front());
2136 auto *BB = Front->getParent();
2137 assert(all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool {
2138 return cast<Instruction>(V)->getParent() == BB;
2141 // The last instruction in the bundle in program order.
2142 Instruction *LastInst = nullptr;
2144 // Find the last instruction. The common case should be that BB has been
2145 // scheduled, and the last instruction is VL.back(). So we start with
2146 // VL.back() and iterate over schedule data until we reach the end of the
2147 // bundle. The end of the bundle is marked by null ScheduleData.
2148 if (BlocksSchedules.count(BB)) {
2149 auto *Bundle = BlocksSchedules[BB]->getScheduleData(VL.back());
2150 if (Bundle && Bundle->isPartOfBundle())
2151 for (; Bundle; Bundle = Bundle->NextInBundle)
2152 LastInst = Bundle->Inst;
2155 // LastInst can still be null at this point if there's either not an entry
2156 // for BB in BlocksSchedules or there's no ScheduleData available for
2157 // VL.back(). This can be the case if buildTree_rec aborts for various
2158 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2159 // size is reached, etc.). ScheduleData is initialized in the scheduling
2162 // If this happens, we can still find the last instruction by brute force. We
2163 // iterate forwards from Front (inclusive) until we either see all
2164 // instructions in the bundle or reach the end of the block. If Front is the
2165 // last instruction in program order, LastInst will be set to Front, and we
2166 // will visit all the remaining instructions in the block.
2168 // One of the reasons we exit early from buildTree_rec is to place an upper
2169 // bound on compile-time. Thus, taking an additional compile-time hit here is
2170 // not ideal. However, this should be exceedingly rare since it requires that
2171 // we both exit early from buildTree_rec and that the bundle be out-of-order
2172 // (causing us to iterate all the way to the end of the block).
2174 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2175 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2176 if (Bundle.erase(&I))
2183 // Set the insertion point after the last instruction in the bundle. Set the
2184 // debug location to Front.
2185 Builder.SetInsertPoint(BB, next(BasicBlock::iterator(LastInst)));
2186 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2189 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2190 Value *Vec = UndefValue::get(Ty);
2191 // Generate the 'InsertElement' instruction.
2192 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2193 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2194 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2195 GatherSeq.insert(Insrt);
2196 CSEBlocks.insert(Insrt->getParent());
2198 // Add to our 'need-to-extract' list.
2199 if (ScalarToTreeEntry.count(VL[i])) {
2200 int Idx = ScalarToTreeEntry[VL[i]];
2201 TreeEntry *E = &VectorizableTree[Idx];
2202 // Find which lane we need to extract.
2204 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2205 // Is this the lane of the scalar that we are looking for ?
2206 if (E->Scalars[Lane] == VL[i]) {
2211 assert(FoundLane >= 0 && "Could not find the correct lane");
2212 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2220 Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL) const {
2221 SmallDenseMap<Value*, int>::const_iterator Entry
2222 = ScalarToTreeEntry.find(VL[0]);
2223 if (Entry != ScalarToTreeEntry.end()) {
2224 int Idx = Entry->second;
2225 const TreeEntry *En = &VectorizableTree[Idx];
2226 if (En->isSame(VL) && En->VectorizedValue)
2227 return En->VectorizedValue;
2232 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2233 if (ScalarToTreeEntry.count(VL[0])) {
2234 int Idx = ScalarToTreeEntry[VL[0]];
2235 TreeEntry *E = &VectorizableTree[Idx];
2237 return vectorizeTree(E);
2240 Type *ScalarTy = VL[0]->getType();
2241 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2242 ScalarTy = SI->getValueOperand()->getType();
2243 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2245 return Gather(VL, VecTy);
2248 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2249 IRBuilder<>::InsertPointGuard Guard(Builder);
2251 if (E->VectorizedValue) {
2252 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
2253 return E->VectorizedValue;
2256 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2257 Type *ScalarTy = VL0->getType();
2258 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2259 ScalarTy = SI->getValueOperand()->getType();
2260 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2262 if (E->NeedToGather) {
2263 setInsertPointAfterBundle(E->Scalars);
2264 auto *V = Gather(E->Scalars, VecTy);
2265 E->VectorizedValue = V;
2269 unsigned Opcode = getSameOpcode(E->Scalars);
2272 case Instruction::PHI: {
2273 PHINode *PH = dyn_cast<PHINode>(VL0);
2274 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2275 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2276 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2277 E->VectorizedValue = NewPhi;
2279 // PHINodes may have multiple entries from the same block. We want to
2280 // visit every block once.
2281 SmallSet<BasicBlock*, 4> VisitedBBs;
2283 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2285 BasicBlock *IBB = PH->getIncomingBlock(i);
2287 if (!VisitedBBs.insert(IBB).second) {
2288 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2292 // Prepare the operand vector.
2293 for (Value *V : E->Scalars)
2294 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2296 Builder.SetInsertPoint(IBB->getTerminator());
2297 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2298 Value *Vec = vectorizeTree(Operands);
2299 NewPhi->addIncoming(Vec, IBB);
2302 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
2303 "Invalid number of incoming values");
2307 case Instruction::ExtractElement: {
2308 if (canReuseExtract(E->Scalars, Instruction::ExtractElement)) {
2309 Value *V = VL0->getOperand(0);
2310 E->VectorizedValue = V;
2313 setInsertPointAfterBundle(E->Scalars);
2314 auto *V = Gather(E->Scalars, VecTy);
2315 E->VectorizedValue = V;
2318 case Instruction::ExtractValue: {
2319 if (canReuseExtract(E->Scalars, Instruction::ExtractValue)) {
2320 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2321 Builder.SetInsertPoint(LI);
2322 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2323 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2324 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2325 E->VectorizedValue = V;
2326 return propagateMetadata(V, E->Scalars);
2328 setInsertPointAfterBundle(E->Scalars);
2329 auto *V = Gather(E->Scalars, VecTy);
2330 E->VectorizedValue = V;
2333 case Instruction::ZExt:
2334 case Instruction::SExt:
2335 case Instruction::FPToUI:
2336 case Instruction::FPToSI:
2337 case Instruction::FPExt:
2338 case Instruction::PtrToInt:
2339 case Instruction::IntToPtr:
2340 case Instruction::SIToFP:
2341 case Instruction::UIToFP:
2342 case Instruction::Trunc:
2343 case Instruction::FPTrunc:
2344 case Instruction::BitCast: {
2346 for (Value *V : E->Scalars)
2347 INVL.push_back(cast<Instruction>(V)->getOperand(0));
2349 setInsertPointAfterBundle(E->Scalars);
2351 Value *InVec = vectorizeTree(INVL);
2353 if (Value *V = alreadyVectorized(E->Scalars))
2356 CastInst *CI = dyn_cast<CastInst>(VL0);
2357 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2358 E->VectorizedValue = V;
2359 ++NumVectorInstructions;
2362 case Instruction::FCmp:
2363 case Instruction::ICmp: {
2364 ValueList LHSV, RHSV;
2365 for (Value *V : E->Scalars) {
2366 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2367 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2370 setInsertPointAfterBundle(E->Scalars);
2372 Value *L = vectorizeTree(LHSV);
2373 Value *R = vectorizeTree(RHSV);
2375 if (Value *V = alreadyVectorized(E->Scalars))
2378 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2380 if (Opcode == Instruction::FCmp)
2381 V = Builder.CreateFCmp(P0, L, R);
2383 V = Builder.CreateICmp(P0, L, R);
2385 E->VectorizedValue = V;
2386 ++NumVectorInstructions;
2389 case Instruction::Select: {
2390 ValueList TrueVec, FalseVec, CondVec;
2391 for (Value *V : E->Scalars) {
2392 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2393 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2394 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2397 setInsertPointAfterBundle(E->Scalars);
2399 Value *Cond = vectorizeTree(CondVec);
2400 Value *True = vectorizeTree(TrueVec);
2401 Value *False = vectorizeTree(FalseVec);
2403 if (Value *V = alreadyVectorized(E->Scalars))
2406 Value *V = Builder.CreateSelect(Cond, True, False);
2407 E->VectorizedValue = V;
2408 ++NumVectorInstructions;
2411 case Instruction::Add:
2412 case Instruction::FAdd:
2413 case Instruction::Sub:
2414 case Instruction::FSub:
2415 case Instruction::Mul:
2416 case Instruction::FMul:
2417 case Instruction::UDiv:
2418 case Instruction::SDiv:
2419 case Instruction::FDiv:
2420 case Instruction::URem:
2421 case Instruction::SRem:
2422 case Instruction::FRem:
2423 case Instruction::Shl:
2424 case Instruction::LShr:
2425 case Instruction::AShr:
2426 case Instruction::And:
2427 case Instruction::Or:
2428 case Instruction::Xor: {
2429 ValueList LHSVL, RHSVL;
2430 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2431 reorderInputsAccordingToOpcode(E->Scalars, LHSVL, RHSVL);
2433 for (Value *V : E->Scalars) {
2434 LHSVL.push_back(cast<Instruction>(V)->getOperand(0));
2435 RHSVL.push_back(cast<Instruction>(V)->getOperand(1));
2438 setInsertPointAfterBundle(E->Scalars);
2440 Value *LHS = vectorizeTree(LHSVL);
2441 Value *RHS = vectorizeTree(RHSVL);
2443 if (LHS == RHS && isa<Instruction>(LHS)) {
2444 assert((VL0->getOperand(0) == VL0->getOperand(1)) && "Invalid order");
2447 if (Value *V = alreadyVectorized(E->Scalars))
2450 BinaryOperator *BinOp = cast<BinaryOperator>(VL0);
2451 Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS);
2452 E->VectorizedValue = V;
2453 propagateIRFlags(E->VectorizedValue, E->Scalars);
2454 ++NumVectorInstructions;
2456 if (Instruction *I = dyn_cast<Instruction>(V))
2457 return propagateMetadata(I, E->Scalars);
2461 case Instruction::Load: {
2462 // Loads are inserted at the head of the tree because we don't want to
2463 // sink them all the way down past store instructions.
2464 setInsertPointAfterBundle(E->Scalars);
2466 LoadInst *LI = cast<LoadInst>(VL0);
2467 Type *ScalarLoadTy = LI->getType();
2468 unsigned AS = LI->getPointerAddressSpace();
2470 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2471 VecTy->getPointerTo(AS));
2473 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2474 // ExternalUses list to make sure that an extract will be generated in the
2476 if (ScalarToTreeEntry.count(LI->getPointerOperand()))
2477 ExternalUses.push_back(
2478 ExternalUser(LI->getPointerOperand(), cast<User>(VecPtr), 0));
2480 unsigned Alignment = LI->getAlignment();
2481 LI = Builder.CreateLoad(VecPtr);
2483 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2485 LI->setAlignment(Alignment);
2486 E->VectorizedValue = LI;
2487 ++NumVectorInstructions;
2488 return propagateMetadata(LI, E->Scalars);
2490 case Instruction::Store: {
2491 StoreInst *SI = cast<StoreInst>(VL0);
2492 unsigned Alignment = SI->getAlignment();
2493 unsigned AS = SI->getPointerAddressSpace();
2496 for (Value *V : E->Scalars)
2497 ValueOp.push_back(cast<StoreInst>(V)->getValueOperand());
2499 setInsertPointAfterBundle(E->Scalars);
2501 Value *VecValue = vectorizeTree(ValueOp);
2502 Value *VecPtr = Builder.CreateBitCast(SI->getPointerOperand(),
2503 VecTy->getPointerTo(AS));
2504 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
2506 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2507 // ExternalUses list to make sure that an extract will be generated in the
2509 if (ScalarToTreeEntry.count(SI->getPointerOperand()))
2510 ExternalUses.push_back(
2511 ExternalUser(SI->getPointerOperand(), cast<User>(VecPtr), 0));
2514 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
2516 S->setAlignment(Alignment);
2517 E->VectorizedValue = S;
2518 ++NumVectorInstructions;
2519 return propagateMetadata(S, E->Scalars);
2521 case Instruction::GetElementPtr: {
2522 setInsertPointAfterBundle(E->Scalars);
2525 for (Value *V : E->Scalars)
2526 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
2528 Value *Op0 = vectorizeTree(Op0VL);
2530 std::vector<Value *> OpVecs;
2531 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
2534 for (Value *V : E->Scalars)
2535 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
2537 Value *OpVec = vectorizeTree(OpVL);
2538 OpVecs.push_back(OpVec);
2541 Value *V = Builder.CreateGEP(
2542 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
2543 E->VectorizedValue = V;
2544 ++NumVectorInstructions;
2546 if (Instruction *I = dyn_cast<Instruction>(V))
2547 return propagateMetadata(I, E->Scalars);
2551 case Instruction::Call: {
2552 CallInst *CI = cast<CallInst>(VL0);
2553 setInsertPointAfterBundle(E->Scalars);
2555 Intrinsic::ID IID = Intrinsic::not_intrinsic;
2556 Value *ScalarArg = nullptr;
2557 if (CI && (FI = CI->getCalledFunction())) {
2558 IID = FI->getIntrinsicID();
2560 std::vector<Value *> OpVecs;
2561 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
2563 // ctlz,cttz and powi are special intrinsics whose second argument is
2564 // a scalar. This argument should not be vectorized.
2565 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
2566 CallInst *CEI = cast<CallInst>(E->Scalars[0]);
2567 ScalarArg = CEI->getArgOperand(j);
2568 OpVecs.push_back(CEI->getArgOperand(j));
2571 for (Value *V : E->Scalars) {
2572 CallInst *CEI = cast<CallInst>(V);
2573 OpVL.push_back(CEI->getArgOperand(j));
2576 Value *OpVec = vectorizeTree(OpVL);
2577 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
2578 OpVecs.push_back(OpVec);
2581 Module *M = F->getParent();
2582 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2583 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
2584 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
2585 SmallVector<OperandBundleDef, 1> OpBundles;
2586 CI->getOperandBundlesAsDefs(OpBundles);
2587 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
2589 // The scalar argument uses an in-tree scalar so we add the new vectorized
2590 // call to ExternalUses list to make sure that an extract will be
2591 // generated in the future.
2592 if (ScalarArg && ScalarToTreeEntry.count(ScalarArg))
2593 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
2595 E->VectorizedValue = V;
2596 ++NumVectorInstructions;
2599 case Instruction::ShuffleVector: {
2600 ValueList LHSVL, RHSVL;
2601 assert(isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand");
2602 reorderAltShuffleOperands(E->Scalars, LHSVL, RHSVL);
2603 setInsertPointAfterBundle(E->Scalars);
2605 Value *LHS = vectorizeTree(LHSVL);
2606 Value *RHS = vectorizeTree(RHSVL);
2608 if (Value *V = alreadyVectorized(E->Scalars))
2611 // Create a vector of LHS op1 RHS
2612 BinaryOperator *BinOp0 = cast<BinaryOperator>(VL0);
2613 Value *V0 = Builder.CreateBinOp(BinOp0->getOpcode(), LHS, RHS);
2615 // Create a vector of LHS op2 RHS
2616 Instruction *VL1 = cast<Instruction>(E->Scalars[1]);
2617 BinaryOperator *BinOp1 = cast<BinaryOperator>(VL1);
2618 Value *V1 = Builder.CreateBinOp(BinOp1->getOpcode(), LHS, RHS);
2620 // Create shuffle to take alternate operations from the vector.
2621 // Also, gather up odd and even scalar ops to propagate IR flags to
2622 // each vector operation.
2623 ValueList OddScalars, EvenScalars;
2624 unsigned e = E->Scalars.size();
2625 SmallVector<Constant *, 8> Mask(e);
2626 for (unsigned i = 0; i < e; ++i) {
2628 Mask[i] = Builder.getInt32(e + i);
2629 OddScalars.push_back(E->Scalars[i]);
2631 Mask[i] = Builder.getInt32(i);
2632 EvenScalars.push_back(E->Scalars[i]);
2636 Value *ShuffleMask = ConstantVector::get(Mask);
2637 propagateIRFlags(V0, EvenScalars);
2638 propagateIRFlags(V1, OddScalars);
2640 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2641 E->VectorizedValue = V;
2642 ++NumVectorInstructions;
2643 if (Instruction *I = dyn_cast<Instruction>(V))
2644 return propagateMetadata(I, E->Scalars);
2649 llvm_unreachable("unknown inst");
2654 Value *BoUpSLP::vectorizeTree() {
2656 // All blocks must be scheduled before any instructions are inserted.
2657 for (auto &BSIter : BlocksSchedules) {
2658 scheduleBlock(BSIter.second.get());
2661 Builder.SetInsertPoint(&F->getEntryBlock().front());
2662 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
2664 // If the vectorized tree can be rewritten in a smaller type, we truncate the
2665 // vectorized root. InstCombine will then rewrite the entire expression. We
2666 // sign extend the extracted values below.
2667 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2668 if (MinBWs.count(ScalarRoot)) {
2669 if (auto *I = dyn_cast<Instruction>(VectorRoot))
2670 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
2671 auto BundleWidth = VectorizableTree[0].Scalars.size();
2672 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot]);
2673 auto *VecTy = VectorType::get(MinTy, BundleWidth);
2674 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
2675 VectorizableTree[0].VectorizedValue = Trunc;
2678 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n");
2680 // Extract all of the elements with the external uses.
2681 for (const auto &ExternalUse : ExternalUses) {
2682 Value *Scalar = ExternalUse.Scalar;
2683 llvm::User *User = ExternalUse.User;
2685 // Skip users that we already RAUW. This happens when one instruction
2686 // has multiple uses of the same value.
2687 if (std::find(Scalar->user_begin(), Scalar->user_end(), User) ==
2690 assert(ScalarToTreeEntry.count(Scalar) && "Invalid scalar");
2692 int Idx = ScalarToTreeEntry[Scalar];
2693 TreeEntry *E = &VectorizableTree[Idx];
2694 assert(!E->NeedToGather && "Extracting from a gather list");
2696 Value *Vec = E->VectorizedValue;
2697 assert(Vec && "Can't find vectorizable value");
2699 Value *Lane = Builder.getInt32(ExternalUse.Lane);
2700 // Generate extracts for out-of-tree users.
2701 // Find the insertion point for the extractelement lane.
2702 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
2703 if (PHINode *PH = dyn_cast<PHINode>(User)) {
2704 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
2705 if (PH->getIncomingValue(i) == Scalar) {
2706 TerminatorInst *IncomingTerminator =
2707 PH->getIncomingBlock(i)->getTerminator();
2708 if (isa<CatchSwitchInst>(IncomingTerminator)) {
2709 Builder.SetInsertPoint(VecI->getParent(),
2710 std::next(VecI->getIterator()));
2712 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
2714 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2715 if (MinBWs.count(ScalarRoot))
2716 Ex = Builder.CreateSExt(Ex, Scalar->getType());
2717 CSEBlocks.insert(PH->getIncomingBlock(i));
2718 PH->setOperand(i, Ex);
2722 Builder.SetInsertPoint(cast<Instruction>(User));
2723 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2724 if (MinBWs.count(ScalarRoot))
2725 Ex = Builder.CreateSExt(Ex, Scalar->getType());
2726 CSEBlocks.insert(cast<Instruction>(User)->getParent());
2727 User->replaceUsesOfWith(Scalar, Ex);
2730 Builder.SetInsertPoint(&F->getEntryBlock().front());
2731 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2732 if (MinBWs.count(ScalarRoot))
2733 Ex = Builder.CreateSExt(Ex, Scalar->getType());
2734 CSEBlocks.insert(&F->getEntryBlock());
2735 User->replaceUsesOfWith(Scalar, Ex);
2738 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
2741 // For each vectorized value:
2742 for (TreeEntry &EIdx : VectorizableTree) {
2743 TreeEntry *Entry = &EIdx;
2746 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
2747 Value *Scalar = Entry->Scalars[Lane];
2748 // No need to handle users of gathered values.
2749 if (Entry->NeedToGather)
2752 assert(Entry->VectorizedValue && "Can't find vectorizable value");
2754 Type *Ty = Scalar->getType();
2755 if (!Ty->isVoidTy()) {
2757 for (User *U : Scalar->users()) {
2758 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
2760 assert((ScalarToTreeEntry.count(U) ||
2761 // It is legal to replace users in the ignorelist by undef.
2762 (std::find(UserIgnoreList.begin(), UserIgnoreList.end(), U) !=
2763 UserIgnoreList.end())) &&
2764 "Replacing out-of-tree value with undef");
2767 Value *Undef = UndefValue::get(Ty);
2768 Scalar->replaceAllUsesWith(Undef);
2770 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
2771 eraseInstruction(cast<Instruction>(Scalar));
2775 Builder.ClearInsertionPoint();
2777 return VectorizableTree[0].VectorizedValue;
2780 void BoUpSLP::optimizeGatherSequence() {
2781 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
2782 << " gather sequences instructions.\n");
2783 // LICM InsertElementInst sequences.
2784 for (Instruction *it : GatherSeq) {
2785 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
2790 // Check if this block is inside a loop.
2791 Loop *L = LI->getLoopFor(Insert->getParent());
2795 // Check if it has a preheader.
2796 BasicBlock *PreHeader = L->getLoopPreheader();
2800 // If the vector or the element that we insert into it are
2801 // instructions that are defined in this basic block then we can't
2802 // hoist this instruction.
2803 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
2804 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
2805 if (CurrVec && L->contains(CurrVec))
2807 if (NewElem && L->contains(NewElem))
2810 // We can hoist this instruction. Move it to the pre-header.
2811 Insert->moveBefore(PreHeader->getTerminator());
2814 // Make a list of all reachable blocks in our CSE queue.
2815 SmallVector<const DomTreeNode *, 8> CSEWorkList;
2816 CSEWorkList.reserve(CSEBlocks.size());
2817 for (BasicBlock *BB : CSEBlocks)
2818 if (DomTreeNode *N = DT->getNode(BB)) {
2819 assert(DT->isReachableFromEntry(N));
2820 CSEWorkList.push_back(N);
2823 // Sort blocks by domination. This ensures we visit a block after all blocks
2824 // dominating it are visited.
2825 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
2826 [this](const DomTreeNode *A, const DomTreeNode *B) {
2827 return DT->properlyDominates(A, B);
2830 // Perform O(N^2) search over the gather sequences and merge identical
2831 // instructions. TODO: We can further optimize this scan if we split the
2832 // instructions into different buckets based on the insert lane.
2833 SmallVector<Instruction *, 16> Visited;
2834 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
2835 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
2836 "Worklist not sorted properly!");
2837 BasicBlock *BB = (*I)->getBlock();
2838 // For all instructions in blocks containing gather sequences:
2839 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
2840 Instruction *In = &*it++;
2841 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
2844 // Check if we can replace this instruction with any of the
2845 // visited instructions.
2846 for (Instruction *v : Visited) {
2847 if (In->isIdenticalTo(v) &&
2848 DT->dominates(v->getParent(), In->getParent())) {
2849 In->replaceAllUsesWith(v);
2850 eraseInstruction(In);
2856 assert(std::find(Visited.begin(), Visited.end(), In) == Visited.end());
2857 Visited.push_back(In);
2865 // Groups the instructions to a bundle (which is then a single scheduling entity)
2866 // and schedules instructions until the bundle gets ready.
2867 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
2869 if (isa<PHINode>(VL[0]))
2872 // Initialize the instruction bundle.
2873 Instruction *OldScheduleEnd = ScheduleEnd;
2874 ScheduleData *PrevInBundle = nullptr;
2875 ScheduleData *Bundle = nullptr;
2876 bool ReSchedule = false;
2877 DEBUG(dbgs() << "SLP: bundle: " << *VL[0] << "\n");
2879 // Make sure that the scheduling region contains all
2880 // instructions of the bundle.
2881 for (Value *V : VL) {
2882 if (!extendSchedulingRegion(V))
2886 for (Value *V : VL) {
2887 ScheduleData *BundleMember = getScheduleData(V);
2888 assert(BundleMember &&
2889 "no ScheduleData for bundle member (maybe not in same basic block)");
2890 if (BundleMember->IsScheduled) {
2891 // A bundle member was scheduled as single instruction before and now
2892 // needs to be scheduled as part of the bundle. We just get rid of the
2893 // existing schedule.
2894 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
2895 << " was already scheduled\n");
2898 assert(BundleMember->isSchedulingEntity() &&
2899 "bundle member already part of other bundle");
2901 PrevInBundle->NextInBundle = BundleMember;
2903 Bundle = BundleMember;
2905 BundleMember->UnscheduledDepsInBundle = 0;
2906 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
2908 // Group the instructions to a bundle.
2909 BundleMember->FirstInBundle = Bundle;
2910 PrevInBundle = BundleMember;
2912 if (ScheduleEnd != OldScheduleEnd) {
2913 // The scheduling region got new instructions at the lower end (or it is a
2914 // new region for the first bundle). This makes it necessary to
2915 // recalculate all dependencies.
2916 // It is seldom that this needs to be done a second time after adding the
2917 // initial bundle to the region.
2918 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
2919 ScheduleData *SD = getScheduleData(I);
2920 SD->clearDependencies();
2926 initialFillReadyList(ReadyInsts);
2929 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
2930 << BB->getName() << "\n");
2932 calculateDependencies(Bundle, true, SLP);
2934 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
2935 // means that there are no cyclic dependencies and we can schedule it.
2936 // Note that's important that we don't "schedule" the bundle yet (see
2937 // cancelScheduling).
2938 while (!Bundle->isReady() && !ReadyInsts.empty()) {
2940 ScheduleData *pickedSD = ReadyInsts.back();
2941 ReadyInsts.pop_back();
2943 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
2944 schedule(pickedSD, ReadyInsts);
2947 if (!Bundle->isReady()) {
2948 cancelScheduling(VL);
2954 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL) {
2955 if (isa<PHINode>(VL[0]))
2958 ScheduleData *Bundle = getScheduleData(VL[0]);
2959 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
2960 assert(!Bundle->IsScheduled &&
2961 "Can't cancel bundle which is already scheduled");
2962 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
2963 "tried to unbundle something which is not a bundle");
2965 // Un-bundle: make single instructions out of the bundle.
2966 ScheduleData *BundleMember = Bundle;
2967 while (BundleMember) {
2968 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
2969 BundleMember->FirstInBundle = BundleMember;
2970 ScheduleData *Next = BundleMember->NextInBundle;
2971 BundleMember->NextInBundle = nullptr;
2972 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
2973 if (BundleMember->UnscheduledDepsInBundle == 0) {
2974 ReadyInsts.insert(BundleMember);
2976 BundleMember = Next;
2980 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V) {
2981 if (getScheduleData(V))
2983 Instruction *I = dyn_cast<Instruction>(V);
2984 assert(I && "bundle member must be an instruction");
2985 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
2986 if (!ScheduleStart) {
2987 // It's the first instruction in the new region.
2988 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
2990 ScheduleEnd = I->getNextNode();
2991 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
2992 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
2995 // Search up and down at the same time, because we don't know if the new
2996 // instruction is above or below the existing scheduling region.
2997 BasicBlock::reverse_iterator UpIter(ScheduleStart->getIterator());
2998 BasicBlock::reverse_iterator UpperEnd = BB->rend();
2999 BasicBlock::iterator DownIter(ScheduleEnd);
3000 BasicBlock::iterator LowerEnd = BB->end();
3002 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3003 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3007 if (UpIter != UpperEnd) {
3008 if (&*UpIter == I) {
3009 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3011 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n");
3016 if (DownIter != LowerEnd) {
3017 if (&*DownIter == I) {
3018 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3020 ScheduleEnd = I->getNextNode();
3021 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3022 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
3027 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
3028 "instruction not found in block");
3033 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3035 ScheduleData *PrevLoadStore,
3036 ScheduleData *NextLoadStore) {
3037 ScheduleData *CurrentLoadStore = PrevLoadStore;
3038 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3039 ScheduleData *SD = ScheduleDataMap[I];
3041 // Allocate a new ScheduleData for the instruction.
3042 if (ChunkPos >= ChunkSize) {
3043 ScheduleDataChunks.push_back(
3044 llvm::make_unique<ScheduleData[]>(ChunkSize));
3047 SD = &(ScheduleDataChunks.back()[ChunkPos++]);
3048 ScheduleDataMap[I] = SD;
3051 assert(!isInSchedulingRegion(SD) &&
3052 "new ScheduleData already in scheduling region");
3053 SD->init(SchedulingRegionID);
3055 if (I->mayReadOrWriteMemory()) {
3056 // Update the linked list of memory accessing instructions.
3057 if (CurrentLoadStore) {
3058 CurrentLoadStore->NextLoadStore = SD;
3060 FirstLoadStoreInRegion = SD;
3062 CurrentLoadStore = SD;
3065 if (NextLoadStore) {
3066 if (CurrentLoadStore)
3067 CurrentLoadStore->NextLoadStore = NextLoadStore;
3069 LastLoadStoreInRegion = CurrentLoadStore;
3073 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3074 bool InsertInReadyList,
3076 assert(SD->isSchedulingEntity());
3078 SmallVector<ScheduleData *, 10> WorkList;
3079 WorkList.push_back(SD);
3081 while (!WorkList.empty()) {
3082 ScheduleData *SD = WorkList.back();
3083 WorkList.pop_back();
3085 ScheduleData *BundleMember = SD;
3086 while (BundleMember) {
3087 assert(isInSchedulingRegion(BundleMember));
3088 if (!BundleMember->hasValidDependencies()) {
3090 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n");
3091 BundleMember->Dependencies = 0;
3092 BundleMember->resetUnscheduledDeps();
3094 // Handle def-use chain dependencies.
3095 for (User *U : BundleMember->Inst->users()) {
3096 if (isa<Instruction>(U)) {
3097 ScheduleData *UseSD = getScheduleData(U);
3098 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3099 BundleMember->Dependencies++;
3100 ScheduleData *DestBundle = UseSD->FirstInBundle;
3101 if (!DestBundle->IsScheduled) {
3102 BundleMember->incrementUnscheduledDeps(1);
3104 if (!DestBundle->hasValidDependencies()) {
3105 WorkList.push_back(DestBundle);
3109 // I'm not sure if this can ever happen. But we need to be safe.
3110 // This lets the instruction/bundle never be scheduled and
3111 // eventually disable vectorization.
3112 BundleMember->Dependencies++;
3113 BundleMember->incrementUnscheduledDeps(1);
3117 // Handle the memory dependencies.
3118 ScheduleData *DepDest = BundleMember->NextLoadStore;
3120 Instruction *SrcInst = BundleMember->Inst;
3121 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3122 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3123 unsigned numAliased = 0;
3124 unsigned DistToSrc = 1;
3127 assert(isInSchedulingRegion(DepDest));
3129 // We have two limits to reduce the complexity:
3130 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3131 // SLP->isAliased (which is the expensive part in this loop).
3132 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3133 // the whole loop (even if the loop is fast, it's quadratic).
3134 // It's important for the loop break condition (see below) to
3135 // check this limit even between two read-only instructions.
3136 if (DistToSrc >= MaxMemDepDistance ||
3137 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3138 (numAliased >= AliasedCheckLimit ||
3139 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3141 // We increment the counter only if the locations are aliased
3142 // (instead of counting all alias checks). This gives a better
3143 // balance between reduced runtime and accurate dependencies.
3146 DepDest->MemoryDependencies.push_back(BundleMember);
3147 BundleMember->Dependencies++;
3148 ScheduleData *DestBundle = DepDest->FirstInBundle;
3149 if (!DestBundle->IsScheduled) {
3150 BundleMember->incrementUnscheduledDeps(1);
3152 if (!DestBundle->hasValidDependencies()) {
3153 WorkList.push_back(DestBundle);
3156 DepDest = DepDest->NextLoadStore;
3158 // Example, explaining the loop break condition: Let's assume our
3159 // starting instruction is i0 and MaxMemDepDistance = 3.
3162 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3165 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3166 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3167 // Previously we already added dependencies from i3 to i6,i7,i8
3168 // (because of MaxMemDepDistance). As we added a dependency from
3169 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3170 // and we can abort this loop at i6.
3171 if (DistToSrc >= 2 * MaxMemDepDistance)
3177 BundleMember = BundleMember->NextInBundle;
3179 if (InsertInReadyList && SD->isReady()) {
3180 ReadyInsts.push_back(SD);
3181 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n");
3186 void BoUpSLP::BlockScheduling::resetSchedule() {
3187 assert(ScheduleStart &&
3188 "tried to reset schedule on block which has not been scheduled");
3189 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3190 ScheduleData *SD = getScheduleData(I);
3191 assert(isInSchedulingRegion(SD));
3192 SD->IsScheduled = false;
3193 SD->resetUnscheduledDeps();
3198 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3200 if (!BS->ScheduleStart)
3203 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
3205 BS->resetSchedule();
3207 // For the real scheduling we use a more sophisticated ready-list: it is
3208 // sorted by the original instruction location. This lets the final schedule
3209 // be as close as possible to the original instruction order.
3210 struct ScheduleDataCompare {
3211 bool operator()(ScheduleData *SD1, ScheduleData *SD2) {
3212 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3215 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3217 // Ensure that all dependency data is updated and fill the ready-list with
3218 // initial instructions.
3220 int NumToSchedule = 0;
3221 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3222 I = I->getNextNode()) {
3223 ScheduleData *SD = BS->getScheduleData(I);
3225 SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) &&
3226 "scheduler and vectorizer have different opinion on what is a bundle");
3227 SD->FirstInBundle->SchedulingPriority = Idx++;
3228 if (SD->isSchedulingEntity()) {
3229 BS->calculateDependencies(SD, false, this);
3233 BS->initialFillReadyList(ReadyInsts);
3235 Instruction *LastScheduledInst = BS->ScheduleEnd;
3237 // Do the "real" scheduling.
3238 while (!ReadyInsts.empty()) {
3239 ScheduleData *picked = *ReadyInsts.begin();
3240 ReadyInsts.erase(ReadyInsts.begin());
3242 // Move the scheduled instruction(s) to their dedicated places, if not
3244 ScheduleData *BundleMember = picked;
3245 while (BundleMember) {
3246 Instruction *pickedInst = BundleMember->Inst;
3247 if (LastScheduledInst->getNextNode() != pickedInst) {
3248 BS->BB->getInstList().remove(pickedInst);
3249 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3252 LastScheduledInst = pickedInst;
3253 BundleMember = BundleMember->NextInBundle;
3256 BS->schedule(picked, ReadyInsts);
3259 assert(NumToSchedule == 0 && "could not schedule all instructions");
3261 // Avoid duplicate scheduling of the block.
3262 BS->ScheduleStart = nullptr;
3265 unsigned BoUpSLP::getVectorElementSize(Value *V) {
3266 // If V is a store, just return the width of the stored value without
3267 // traversing the expression tree. This is the common case.
3268 if (auto *Store = dyn_cast<StoreInst>(V))
3269 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3271 // If V is not a store, we can traverse the expression tree to find loads
3272 // that feed it. The type of the loaded value may indicate a more suitable
3273 // width than V's type. We want to base the vector element size on the width
3274 // of memory operations where possible.
3275 SmallVector<Instruction *, 16> Worklist;
3276 SmallPtrSet<Instruction *, 16> Visited;
3277 if (auto *I = dyn_cast<Instruction>(V))
3278 Worklist.push_back(I);
3280 // Traverse the expression tree in bottom-up order looking for loads. If we
3281 // encounter an instruciton we don't yet handle, we give up.
3283 auto FoundUnknownInst = false;
3284 while (!Worklist.empty() && !FoundUnknownInst) {
3285 auto *I = Worklist.pop_back_val();
3288 // We should only be looking at scalar instructions here. If the current
3289 // instruction has a vector type, give up.
3290 auto *Ty = I->getType();
3291 if (isa<VectorType>(Ty))
3292 FoundUnknownInst = true;
3294 // If the current instruction is a load, update MaxWidth to reflect the
3295 // width of the loaded value.
3296 else if (isa<LoadInst>(I))
3297 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3299 // Otherwise, we need to visit the operands of the instruction. We only
3300 // handle the interesting cases from buildTree here. If an operand is an
3301 // instruction we haven't yet visited, we add it to the worklist.
3302 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3303 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3304 for (Use &U : I->operands())
3305 if (auto *J = dyn_cast<Instruction>(U.get()))
3306 if (!Visited.count(J))
3307 Worklist.push_back(J);
3310 // If we don't yet handle the instruction, give up.
3312 FoundUnknownInst = true;
3315 // If we didn't encounter a memory access in the expression tree, or if we
3316 // gave up for some reason, just return the width of V.
3317 if (!MaxWidth || FoundUnknownInst)
3318 return DL->getTypeSizeInBits(V->getType());
3320 // Otherwise, return the maximum width we found.
3324 // Determine if a value V in a vectorizable expression Expr can be demoted to a
3325 // smaller type with a truncation. We collect the values that will be demoted
3326 // in ToDemote and additional roots that require investigating in Roots.
3327 static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
3328 SmallVectorImpl<Value *> &ToDemote,
3329 SmallVectorImpl<Value *> &Roots) {
3331 // We can always demote constants.
3332 if (isa<Constant>(V)) {
3333 ToDemote.push_back(V);
3337 // If the value is not an instruction in the expression with only one use, it
3338 // cannot be demoted.
3339 auto *I = dyn_cast<Instruction>(V);
3340 if (!I || !I->hasOneUse() || !Expr.count(I))
3343 switch (I->getOpcode()) {
3345 // We can always demote truncations and extensions. Since truncations can
3346 // seed additional demotion, we save the truncated value.
3347 case Instruction::Trunc:
3348 Roots.push_back(I->getOperand(0));
3349 case Instruction::ZExt:
3350 case Instruction::SExt:
3353 // We can demote certain binary operations if we can demote both of their
3355 case Instruction::Add:
3356 case Instruction::Sub:
3357 case Instruction::Mul:
3358 case Instruction::And:
3359 case Instruction::Or:
3360 case Instruction::Xor:
3361 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3362 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3366 // We can demote selects if we can demote their true and false values.
3367 case Instruction::Select: {
3368 SelectInst *SI = cast<SelectInst>(I);
3369 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3370 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3375 // We can demote phis if we can demote all their incoming operands. Note that
3376 // we don't need to worry about cycles since we ensure single use above.
3377 case Instruction::PHI: {
3378 PHINode *PN = cast<PHINode>(I);
3379 for (Value *IncValue : PN->incoming_values())
3380 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3385 // Otherwise, conservatively give up.
3390 // Record the value that we can demote.
3391 ToDemote.push_back(V);
3395 void BoUpSLP::computeMinimumValueSizes() {
3396 // If there are no external uses, the expression tree must be rooted by a
3397 // store. We can't demote in-memory values, so there is nothing to do here.
3398 if (ExternalUses.empty())
3401 // We only attempt to truncate integer expressions.
3402 auto &TreeRoot = VectorizableTree[0].Scalars;
3403 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3407 // If the expression is not rooted by a store, these roots should have
3408 // external uses. We will rely on InstCombine to rewrite the expression in
3409 // the narrower type. However, InstCombine only rewrites single-use values.
3410 // This means that if a tree entry other than a root is used externally, it
3411 // must have multiple uses and InstCombine will not rewrite it. The code
3412 // below ensures that only the roots are used externally.
3413 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3414 for (auto &EU : ExternalUses)
3415 if (!Expr.erase(EU.Scalar))
3420 // Collect the scalar values of the vectorizable expression. We will use this
3421 // context to determine which values can be demoted. If we see a truncation,
3422 // we mark it as seeding another demotion.
3423 for (auto &Entry : VectorizableTree)
3424 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3426 // Ensure the roots of the vectorizable tree don't form a cycle. They must
3427 // have a single external user that is not in the vectorizable tree.
3428 for (auto *Root : TreeRoot)
3429 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
3432 // Conservatively determine if we can actually truncate the roots of the
3433 // expression. Collect the values that can be demoted in ToDemote and
3434 // additional roots that require investigating in Roots.
3435 SmallVector<Value *, 32> ToDemote;
3436 SmallVector<Value *, 4> Roots;
3437 for (auto *Root : TreeRoot)
3438 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
3441 // The maximum bit width required to represent all the values that can be
3442 // demoted without loss of precision. It would be safe to truncate the roots
3443 // of the expression to this width.
3444 auto MaxBitWidth = 8u;
3446 // We first check if all the bits of the roots are demanded. If they're not,
3447 // we can truncate the roots to this narrower type.
3448 for (auto *Root : TreeRoot) {
3449 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
3450 MaxBitWidth = std::max<unsigned>(
3451 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
3454 // If all the bits of the roots are demanded, we can try a little harder to
3455 // compute a narrower type. This can happen, for example, if the roots are
3456 // getelementptr indices. InstCombine promotes these indices to the pointer
3457 // width. Thus, all their bits are technically demanded even though the
3458 // address computation might be vectorized in a smaller type.
3460 // We start by looking at each entry that can be demoted. We compute the
3461 // maximum bit width required to store the scalar by using ValueTracking to
3462 // compute the number of high-order bits we can truncate.
3463 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
3465 for (auto *Scalar : ToDemote) {
3466 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, 0, DT);
3467 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
3468 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
3472 // Round MaxBitWidth up to the next power-of-two.
3473 if (!isPowerOf2_64(MaxBitWidth))
3474 MaxBitWidth = NextPowerOf2(MaxBitWidth);
3476 // If the maximum bit width we compute is less than the with of the roots'
3477 // type, we can proceed with the narrowing. Otherwise, do nothing.
3478 if (MaxBitWidth >= TreeRootIT->getBitWidth())
3481 // If we can truncate the root, we must collect additional values that might
3482 // be demoted as a result. That is, those seeded by truncations we will
3484 while (!Roots.empty())
3485 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
3487 // Finally, map the values we can demote to the maximum bit with we computed.
3488 for (auto *Scalar : ToDemote)
3489 MinBWs[Scalar] = MaxBitWidth;
3493 /// The SLPVectorizer Pass.
3494 struct SLPVectorizer : public FunctionPass {
3495 SLPVectorizerPass Impl;
3497 /// Pass identification, replacement for typeid
3500 explicit SLPVectorizer() : FunctionPass(ID) {
3501 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
3505 bool doInitialization(Module &M) override {
3509 bool runOnFunction(Function &F) override {
3510 if (skipFunction(F))
3513 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
3514 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
3515 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
3516 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
3517 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3518 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
3519 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
3520 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
3521 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
3523 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB);
3526 void getAnalysisUsage(AnalysisUsage &AU) const override {
3527 FunctionPass::getAnalysisUsage(AU);
3528 AU.addRequired<AssumptionCacheTracker>();
3529 AU.addRequired<ScalarEvolutionWrapperPass>();
3530 AU.addRequired<AAResultsWrapperPass>();
3531 AU.addRequired<TargetTransformInfoWrapperPass>();
3532 AU.addRequired<LoopInfoWrapperPass>();
3533 AU.addRequired<DominatorTreeWrapperPass>();
3534 AU.addRequired<DemandedBitsWrapperPass>();
3535 AU.addPreserved<LoopInfoWrapperPass>();
3536 AU.addPreserved<DominatorTreeWrapperPass>();
3537 AU.addPreserved<AAResultsWrapperPass>();
3538 AU.addPreserved<GlobalsAAWrapperPass>();
3539 AU.setPreservesCFG();
3542 } // end anonymous namespace
3544 PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
3545 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
3546 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
3547 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
3548 auto *AA = &AM.getResult<AAManager>(F);
3549 auto *LI = &AM.getResult<LoopAnalysis>(F);
3550 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
3551 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
3552 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
3554 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB);
3556 return PreservedAnalyses::all();
3557 PreservedAnalyses PA;
3558 PA.preserve<LoopAnalysis>();
3559 PA.preserve<DominatorTreeAnalysis>();
3560 PA.preserve<AAManager>();
3561 PA.preserve<GlobalsAA>();
3565 bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
3566 TargetTransformInfo *TTI_,
3567 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
3568 LoopInfo *LI_, DominatorTree *DT_,
3569 AssumptionCache *AC_, DemandedBits *DB_) {
3578 DL = &F.getParent()->getDataLayout();
3582 bool Changed = false;
3584 // If the target claims to have no vector registers don't attempt
3586 if (!TTI->getNumberOfRegisters(true))
3589 // Don't vectorize when the attribute NoImplicitFloat is used.
3590 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
3593 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
3595 // Use the bottom up slp vectorizer to construct chains that start with
3596 // store instructions.
3597 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL);
3599 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
3600 // delete instructions.
3602 // Scan the blocks in the function in post order.
3603 for (auto BB : post_order(&F.getEntryBlock())) {
3604 collectSeedInstructions(BB);
3606 // Vectorize trees that end at stores.
3607 if (!Stores.empty()) {
3608 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
3609 << " underlying objects.\n");
3610 Changed |= vectorizeStoreChains(R);
3613 // Vectorize trees that end at reductions.
3614 Changed |= vectorizeChainsInBlock(BB, R);
3616 // Vectorize the index computations of getelementptr instructions. This
3617 // is primarily intended to catch gather-like idioms ending at
3618 // non-consecutive loads.
3619 if (!GEPs.empty()) {
3620 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
3621 << " underlying objects.\n");
3622 Changed |= vectorizeGEPIndices(BB, R);
3627 R.optimizeGatherSequence();
3628 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
3629 DEBUG(verifyFunction(F));
3634 /// \brief Check that the Values in the slice in VL array are still existent in
3635 /// the WeakVH array.
3636 /// Vectorization of part of the VL array may cause later values in the VL array
3637 /// to become invalid. We track when this has happened in the WeakVH array.
3638 static bool hasValueBeenRAUWed(ArrayRef<Value *> VL, ArrayRef<WeakVH> VH,
3639 unsigned SliceBegin, unsigned SliceSize) {
3640 VL = VL.slice(SliceBegin, SliceSize);
3641 VH = VH.slice(SliceBegin, SliceSize);
3642 return !std::equal(VL.begin(), VL.end(), VH.begin());
3645 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain,
3646 int CostThreshold, BoUpSLP &R,
3647 unsigned VecRegSize) {
3648 unsigned ChainLen = Chain.size();
3649 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
3651 unsigned Sz = R.getVectorElementSize(Chain[0]);
3652 unsigned VF = VecRegSize / Sz;
3654 if (!isPowerOf2_32(Sz) || VF < 2)
3657 // Keep track of values that were deleted by vectorizing in the loop below.
3658 SmallVector<WeakVH, 8> TrackValues(Chain.begin(), Chain.end());
3660 bool Changed = false;
3661 // Look for profitable vectorizable trees at all offsets, starting at zero.
3662 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
3666 // Check that a previous iteration of this loop did not delete the Value.
3667 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
3670 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
3672 ArrayRef<Value *> Operands = Chain.slice(i, VF);
3674 R.buildTree(Operands);
3675 R.computeMinimumValueSizes();
3677 int Cost = R.getTreeCost();
3679 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
3680 if (Cost < CostThreshold) {
3681 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
3684 // Move to the next bundle.
3693 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
3694 int costThreshold, BoUpSLP &R) {
3695 SetVector<StoreInst *> Heads, Tails;
3696 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
3698 // We may run into multiple chains that merge into a single chain. We mark the
3699 // stores that we vectorized so that we don't visit the same store twice.
3700 BoUpSLP::ValueSet VectorizedStores;
3701 bool Changed = false;
3703 // Do a quadratic search on all of the given stores and find
3704 // all of the pairs of stores that follow each other.
3705 SmallVector<unsigned, 16> IndexQueue;
3706 for (unsigned i = 0, e = Stores.size(); i < e; ++i) {
3708 // If a store has multiple consecutive store candidates, search Stores
3709 // array according to the sequence: from i+1 to e, then from i-1 to 0.
3710 // This is because usually pairing with immediate succeeding or preceding
3711 // candidate create the best chance to find slp vectorization opportunity.
3713 for (j = i + 1; j < e; ++j)
3714 IndexQueue.push_back(j);
3715 for (j = i; j > 0; --j)
3716 IndexQueue.push_back(j - 1);
3718 for (auto &k : IndexQueue) {
3719 if (isConsecutiveAccess(Stores[i], Stores[k], *DL, *SE)) {
3720 Tails.insert(Stores[k]);
3721 Heads.insert(Stores[i]);
3722 ConsecutiveChain[Stores[i]] = Stores[k];
3728 // For stores that start but don't end a link in the chain:
3729 for (SetVector<StoreInst *>::iterator it = Heads.begin(), e = Heads.end();
3731 if (Tails.count(*it))
3734 // We found a store instr that starts a chain. Now follow the chain and try
3736 BoUpSLP::ValueList Operands;
3738 // Collect the chain into a list.
3739 while (Tails.count(I) || Heads.count(I)) {
3740 if (VectorizedStores.count(I))
3742 Operands.push_back(I);
3743 // Move to the next value in the chain.
3744 I = ConsecutiveChain[I];
3747 // FIXME: Is division-by-2 the correct step? Should we assert that the
3748 // register size is a power-of-2?
3749 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize(); Size /= 2) {
3750 if (vectorizeStoreChain(Operands, costThreshold, R, Size)) {
3751 // Mark the vectorized stores so that we don't vectorize them again.
3752 VectorizedStores.insert(Operands.begin(), Operands.end());
3762 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
3764 // Initialize the collections. We will make a single pass over the block.
3768 // Visit the store and getelementptr instructions in BB and organize them in
3769 // Stores and GEPs according to the underlying objects of their pointer
3771 for (Instruction &I : *BB) {
3773 // Ignore store instructions that are volatile or have a pointer operand
3774 // that doesn't point to a scalar type.
3775 if (auto *SI = dyn_cast<StoreInst>(&I)) {
3776 if (!SI->isSimple())
3778 if (!isValidElementType(SI->getValueOperand()->getType()))
3780 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
3783 // Ignore getelementptr instructions that have more than one index, a
3784 // constant index, or a pointer operand that doesn't point to a scalar
3786 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
3787 auto Idx = GEP->idx_begin()->get();
3788 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
3790 if (!isValidElementType(Idx->getType()))
3792 if (GEP->getType()->isVectorTy())
3794 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
3799 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
3802 Value *VL[] = { A, B };
3803 return tryToVectorizeList(VL, R, None, true);
3806 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
3807 ArrayRef<Value *> BuildVector,
3808 bool allowReorder) {
3812 DEBUG(dbgs() << "SLP: Vectorizing a list of length = " << VL.size() << ".\n");
3814 // Check that all of the parts are scalar instructions of the same type.
3815 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
3819 unsigned Opcode0 = I0->getOpcode();
3821 // FIXME: Register size should be a parameter to this function, so we can
3822 // try different vectorization factors.
3823 unsigned Sz = R.getVectorElementSize(I0);
3824 unsigned VF = R.getMinVecRegSize() / Sz;
3826 for (Value *V : VL) {
3827 Type *Ty = V->getType();
3828 if (!isValidElementType(Ty))
3830 Instruction *Inst = dyn_cast<Instruction>(V);
3831 if (!Inst || Inst->getOpcode() != Opcode0)
3835 bool Changed = false;
3837 // Keep track of values that were deleted by vectorizing in the loop below.
3838 SmallVector<WeakVH, 8> TrackValues(VL.begin(), VL.end());
3840 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
3841 unsigned OpsWidth = 0;
3848 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
3851 // Check that a previous iteration of this loop did not delete the Value.
3852 if (hasValueBeenRAUWed(VL, TrackValues, i, OpsWidth))
3855 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
3857 ArrayRef<Value *> Ops = VL.slice(i, OpsWidth);
3859 ArrayRef<Value *> BuildVectorSlice;
3860 if (!BuildVector.empty())
3861 BuildVectorSlice = BuildVector.slice(i, OpsWidth);
3863 R.buildTree(Ops, BuildVectorSlice);
3864 // TODO: check if we can allow reordering also for other cases than
3865 // tryToVectorizePair()
3866 if (allowReorder && R.shouldReorder()) {
3867 assert(Ops.size() == 2);
3868 assert(BuildVectorSlice.empty());
3869 Value *ReorderedOps[] = { Ops[1], Ops[0] };
3870 R.buildTree(ReorderedOps, None);
3872 R.computeMinimumValueSizes();
3873 int Cost = R.getTreeCost();
3875 if (Cost < -SLPCostThreshold) {
3876 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
3877 Value *VectorizedRoot = R.vectorizeTree();
3879 // Reconstruct the build vector by extracting the vectorized root. This
3880 // way we handle the case where some elements of the vector are undefined.
3881 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
3882 if (!BuildVectorSlice.empty()) {
3883 // The insert point is the last build vector instruction. The vectorized
3884 // root will precede it. This guarantees that we get an instruction. The
3885 // vectorized tree could have been constant folded.
3886 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
3887 unsigned VecIdx = 0;
3888 for (auto &V : BuildVectorSlice) {
3889 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
3890 ++BasicBlock::iterator(InsertAfter));
3891 Instruction *I = cast<Instruction>(V);
3892 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I));
3893 Instruction *Extract = cast<Instruction>(Builder.CreateExtractElement(
3894 VectorizedRoot, Builder.getInt32(VecIdx++)));
3895 I->setOperand(1, Extract);
3896 I->removeFromParent();
3897 I->insertAfter(Extract);
3901 // Move to the next bundle.
3910 bool SLPVectorizerPass::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
3914 // Try to vectorize V.
3915 if (tryToVectorizePair(V->getOperand(0), V->getOperand(1), R))
3918 BinaryOperator *A = dyn_cast<BinaryOperator>(V->getOperand(0));
3919 BinaryOperator *B = dyn_cast<BinaryOperator>(V->getOperand(1));
3921 if (B && B->hasOneUse()) {
3922 BinaryOperator *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
3923 BinaryOperator *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
3924 if (tryToVectorizePair(A, B0, R)) {
3927 if (tryToVectorizePair(A, B1, R)) {
3933 if (A && A->hasOneUse()) {
3934 BinaryOperator *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
3935 BinaryOperator *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
3936 if (tryToVectorizePair(A0, B, R)) {
3939 if (tryToVectorizePair(A1, B, R)) {
3946 /// \brief Generate a shuffle mask to be used in a reduction tree.
3948 /// \param VecLen The length of the vector to be reduced.
3949 /// \param NumEltsToRdx The number of elements that should be reduced in the
3951 /// \param IsPairwise Whether the reduction is a pairwise or splitting
3952 /// reduction. A pairwise reduction will generate a mask of
3953 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
3954 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
3955 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
3956 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
3957 bool IsPairwise, bool IsLeft,
3958 IRBuilder<> &Builder) {
3959 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
3961 SmallVector<Constant *, 32> ShuffleMask(
3962 VecLen, UndefValue::get(Builder.getInt32Ty()));
3965 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
3966 for (unsigned i = 0; i != NumEltsToRdx; ++i)
3967 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
3969 // Move the upper half of the vector to the lower half.
3970 for (unsigned i = 0; i != NumEltsToRdx; ++i)
3971 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
3973 return ConstantVector::get(ShuffleMask);
3977 /// Model horizontal reductions.
3979 /// A horizontal reduction is a tree of reduction operations (currently add and
3980 /// fadd) that has operations that can be put into a vector as its leaf.
3981 /// For example, this tree:
3988 /// This tree has "mul" as its reduced values and "+" as its reduction
3989 /// operations. A reduction might be feeding into a store or a binary operation
4004 class HorizontalReduction {
4005 SmallVector<Value *, 16> ReductionOps;
4006 SmallVector<Value *, 32> ReducedVals;
4008 BinaryOperator *ReductionRoot;
4009 PHINode *ReductionPHI;
4011 /// The opcode of the reduction.
4012 unsigned ReductionOpcode;
4013 /// The opcode of the values we perform a reduction on.
4014 unsigned ReducedValueOpcode;
4015 /// Should we model this reduction as a pairwise reduction tree or a tree that
4016 /// splits the vector in halves and adds those halves.
4017 bool IsPairwiseReduction;
4020 /// The width of one full horizontal reduction operation.
4021 unsigned ReduxWidth;
4023 /// Minimal width of available vector registers. It's used to determine
4025 unsigned MinVecRegSize;
4027 HorizontalReduction(unsigned MinVecRegSize)
4028 : ReductionRoot(nullptr), ReductionPHI(nullptr), ReductionOpcode(0),
4029 ReducedValueOpcode(0), IsPairwiseReduction(false), ReduxWidth(0),
4030 MinVecRegSize(MinVecRegSize) {}
4032 /// \brief Try to find a reduction tree.
4033 bool matchAssociativeReduction(PHINode *Phi, BinaryOperator *B) {
4035 std::find(Phi->op_begin(), Phi->op_end(), B) != Phi->op_end()) &&
4036 "Thi phi needs to use the binary operator");
4038 // We could have a initial reductions that is not an add.
4039 // r *= v1 + v2 + v3 + v4
4040 // In such a case start looking for a tree rooted in the first '+'.
4042 if (B->getOperand(0) == Phi) {
4044 B = dyn_cast<BinaryOperator>(B->getOperand(1));
4045 } else if (B->getOperand(1) == Phi) {
4047 B = dyn_cast<BinaryOperator>(B->getOperand(0));
4054 Type *Ty = B->getType();
4055 if (!isValidElementType(Ty))
4058 const DataLayout &DL = B->getModule()->getDataLayout();
4059 ReductionOpcode = B->getOpcode();
4060 ReducedValueOpcode = 0;
4061 // FIXME: Register size should be a parameter to this function, so we can
4062 // try different vectorization factors.
4063 ReduxWidth = MinVecRegSize / DL.getTypeSizeInBits(Ty);
4070 // We currently only support adds.
4071 if (ReductionOpcode != Instruction::Add &&
4072 ReductionOpcode != Instruction::FAdd)
4075 // Post order traverse the reduction tree starting at B. We only handle true
4076 // trees containing only binary operators or selects.
4077 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
4078 Stack.push_back(std::make_pair(B, 0));
4079 while (!Stack.empty()) {
4080 Instruction *TreeN = Stack.back().first;
4081 unsigned EdgeToVist = Stack.back().second++;
4082 bool IsReducedValue = TreeN->getOpcode() != ReductionOpcode;
4084 // Only handle trees in the current basic block.
4085 if (TreeN->getParent() != B->getParent())
4088 // Each tree node needs to have one user except for the ultimate
4090 if (!TreeN->hasOneUse() && TreeN != B)
4094 if (EdgeToVist == 2 || IsReducedValue) {
4095 if (IsReducedValue) {
4096 // Make sure that the opcodes of the operations that we are going to
4098 if (!ReducedValueOpcode)
4099 ReducedValueOpcode = TreeN->getOpcode();
4100 else if (ReducedValueOpcode != TreeN->getOpcode())
4102 ReducedVals.push_back(TreeN);
4104 // We need to be able to reassociate the adds.
4105 if (!TreeN->isAssociative())
4107 ReductionOps.push_back(TreeN);
4114 // Visit left or right.
4115 Value *NextV = TreeN->getOperand(EdgeToVist);
4116 // We currently only allow BinaryOperator's and SelectInst's as reduction
4117 // values in our tree.
4118 if (isa<BinaryOperator>(NextV) || isa<SelectInst>(NextV))
4119 Stack.push_back(std::make_pair(cast<Instruction>(NextV), 0));
4120 else if (NextV != Phi)
4126 /// \brief Attempt to vectorize the tree found by
4127 /// matchAssociativeReduction.
4128 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
4129 if (ReducedVals.empty())
4132 unsigned NumReducedVals = ReducedVals.size();
4133 if (NumReducedVals < ReduxWidth)
4136 Value *VectorizedTree = nullptr;
4137 IRBuilder<> Builder(ReductionRoot);
4138 FastMathFlags Unsafe;
4139 Unsafe.setUnsafeAlgebra();
4140 Builder.setFastMathFlags(Unsafe);
4143 for (; i < NumReducedVals - ReduxWidth + 1; i += ReduxWidth) {
4144 V.buildTree(makeArrayRef(&ReducedVals[i], ReduxWidth), ReductionOps);
4145 V.computeMinimumValueSizes();
4148 int Cost = V.getTreeCost() + getReductionCost(TTI, ReducedVals[i]);
4149 if (Cost >= -SLPCostThreshold)
4152 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Cost
4155 // Vectorize a tree.
4156 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
4157 Value *VectorizedRoot = V.vectorizeTree();
4159 // Emit a reduction.
4160 Value *ReducedSubTree = emitReduction(VectorizedRoot, Builder);
4161 if (VectorizedTree) {
4162 Builder.SetCurrentDebugLocation(Loc);
4163 VectorizedTree = createBinOp(Builder, ReductionOpcode, VectorizedTree,
4164 ReducedSubTree, "bin.rdx");
4166 VectorizedTree = ReducedSubTree;
4169 if (VectorizedTree) {
4170 // Finish the reduction.
4171 for (; i < NumReducedVals; ++i) {
4172 Builder.SetCurrentDebugLocation(
4173 cast<Instruction>(ReducedVals[i])->getDebugLoc());
4174 VectorizedTree = createBinOp(Builder, ReductionOpcode, VectorizedTree,
4179 assert(ReductionRoot && "Need a reduction operation");
4180 ReductionRoot->setOperand(0, VectorizedTree);
4181 ReductionRoot->setOperand(1, ReductionPHI);
4183 ReductionRoot->replaceAllUsesWith(VectorizedTree);
4185 return VectorizedTree != nullptr;
4188 unsigned numReductionValues() const {
4189 return ReducedVals.size();
4193 /// \brief Calculate the cost of a reduction.
4194 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal) {
4195 Type *ScalarTy = FirstReducedVal->getType();
4196 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
4198 int PairwiseRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, true);
4199 int SplittingRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, false);
4201 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
4202 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
4204 int ScalarReduxCost =
4205 ReduxWidth * TTI->getArithmeticInstrCost(ReductionOpcode, VecTy);
4207 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost
4208 << " for reduction that starts with " << *FirstReducedVal
4210 << (IsPairwiseReduction ? "pairwise" : "splitting")
4211 << " reduction)\n");
4213 return VecReduxCost - ScalarReduxCost;
4216 static Value *createBinOp(IRBuilder<> &Builder, unsigned Opcode, Value *L,
4217 Value *R, const Twine &Name = "") {
4218 if (Opcode == Instruction::FAdd)
4219 return Builder.CreateFAdd(L, R, Name);
4220 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, L, R, Name);
4223 /// \brief Emit a horizontal reduction of the vectorized value.
4224 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder) {
4225 assert(VectorizedValue && "Need to have a vectorized tree node");
4226 assert(isPowerOf2_32(ReduxWidth) &&
4227 "We only handle power-of-two reductions for now");
4229 Value *TmpVec = VectorizedValue;
4230 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
4231 if (IsPairwiseReduction) {
4233 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
4235 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
4237 Value *LeftShuf = Builder.CreateShuffleVector(
4238 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
4239 Value *RightShuf = Builder.CreateShuffleVector(
4240 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
4242 TmpVec = createBinOp(Builder, ReductionOpcode, LeftShuf, RightShuf,
4246 createRdxShuffleMask(ReduxWidth, i, false, false, Builder);
4247 Value *Shuf = Builder.CreateShuffleVector(
4248 TmpVec, UndefValue::get(TmpVec->getType()), UpperHalf, "rdx.shuf");
4249 TmpVec = createBinOp(Builder, ReductionOpcode, TmpVec, Shuf, "bin.rdx");
4253 // The result is in the first element of the vector.
4254 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
4258 /// \brief Recognize construction of vectors like
4259 /// %ra = insertelement <4 x float> undef, float %s0, i32 0
4260 /// %rb = insertelement <4 x float> %ra, float %s1, i32 1
4261 /// %rc = insertelement <4 x float> %rb, float %s2, i32 2
4262 /// %rd = insertelement <4 x float> %rc, float %s3, i32 3
4264 /// Returns true if it matches
4266 static bool findBuildVector(InsertElementInst *FirstInsertElem,
4267 SmallVectorImpl<Value *> &BuildVector,
4268 SmallVectorImpl<Value *> &BuildVectorOpds) {
4269 if (!isa<UndefValue>(FirstInsertElem->getOperand(0)))
4272 InsertElementInst *IE = FirstInsertElem;
4274 BuildVector.push_back(IE);
4275 BuildVectorOpds.push_back(IE->getOperand(1));
4277 if (IE->use_empty())
4280 InsertElementInst *NextUse = dyn_cast<InsertElementInst>(IE->user_back());
4284 // If this isn't the final use, make sure the next insertelement is the only
4285 // use. It's OK if the final constructed vector is used multiple times
4286 if (!IE->hasOneUse())
4295 /// \brief Like findBuildVector, but looks backwards for construction of aggregate.
4297 /// \return true if it matches.
4298 static bool findBuildAggregate(InsertValueInst *IV,
4299 SmallVectorImpl<Value *> &BuildVector,
4300 SmallVectorImpl<Value *> &BuildVectorOpds) {
4301 if (!IV->hasOneUse())
4303 Value *V = IV->getAggregateOperand();
4304 if (!isa<UndefValue>(V)) {
4305 InsertValueInst *I = dyn_cast<InsertValueInst>(V);
4306 if (!I || !findBuildAggregate(I, BuildVector, BuildVectorOpds))
4309 BuildVector.push_back(IV);
4310 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
4314 static bool PhiTypeSorterFunc(Value *V, Value *V2) {
4315 return V->getType() < V2->getType();
4318 /// \brief Try and get a reduction value from a phi node.
4320 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions
4321 /// if they come from either \p ParentBB or a containing loop latch.
4323 /// \returns A candidate reduction value if possible, or \code nullptr \endcode
4324 /// if not possible.
4325 static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
4326 BasicBlock *ParentBB, LoopInfo *LI) {
4327 // There are situations where the reduction value is not dominated by the
4328 // reduction phi. Vectorizing such cases has been reported to cause
4329 // miscompiles. See PR25787.
4330 auto DominatedReduxValue = [&](Value *R) {
4332 dyn_cast<Instruction>(R) &&
4333 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
4336 Value *Rdx = nullptr;
4338 // Return the incoming value if it comes from the same BB as the phi node.
4339 if (P->getIncomingBlock(0) == ParentBB) {
4340 Rdx = P->getIncomingValue(0);
4341 } else if (P->getIncomingBlock(1) == ParentBB) {
4342 Rdx = P->getIncomingValue(1);
4345 if (Rdx && DominatedReduxValue(Rdx))
4348 // Otherwise, check whether we have a loop latch to look at.
4349 Loop *BBL = LI->getLoopFor(ParentBB);
4352 BasicBlock *BBLatch = BBL->getLoopLatch();
4356 // There is a loop latch, return the incoming value if it comes from
4357 // that. This reduction pattern occassionaly turns up.
4358 if (P->getIncomingBlock(0) == BBLatch) {
4359 Rdx = P->getIncomingValue(0);
4360 } else if (P->getIncomingBlock(1) == BBLatch) {
4361 Rdx = P->getIncomingValue(1);
4364 if (Rdx && DominatedReduxValue(Rdx))
4370 /// \brief Attempt to reduce a horizontal reduction.
4371 /// If it is legal to match a horizontal reduction feeding
4372 /// the phi node P with reduction operators BI, then check if it
4374 /// \returns true if a horizontal reduction was matched and reduced.
4375 /// \returns false if a horizontal reduction was not matched.
4376 static bool canMatchHorizontalReduction(PHINode *P, BinaryOperator *BI,
4377 BoUpSLP &R, TargetTransformInfo *TTI,
4378 unsigned MinRegSize) {
4379 if (!ShouldVectorizeHor)
4382 HorizontalReduction HorRdx(MinRegSize);
4383 if (!HorRdx.matchAssociativeReduction(P, BI))
4386 // If there is a sufficient number of reduction values, reduce
4387 // to a nearby power-of-2. Can safely generate oversized
4388 // vectors and rely on the backend to split them to legal sizes.
4390 std::max((uint64_t)4, PowerOf2Floor(HorRdx.numReductionValues()));
4392 return HorRdx.tryToReduce(R, TTI);
4395 bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
4396 bool Changed = false;
4397 SmallVector<Value *, 4> Incoming;
4398 SmallSet<Value *, 16> VisitedInstrs;
4400 bool HaveVectorizedPhiNodes = true;
4401 while (HaveVectorizedPhiNodes) {
4402 HaveVectorizedPhiNodes = false;
4404 // Collect the incoming values from the PHIs.
4406 for (Instruction &I : *BB) {
4407 PHINode *P = dyn_cast<PHINode>(&I);
4411 if (!VisitedInstrs.count(P))
4412 Incoming.push_back(P);
4416 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
4418 // Try to vectorize elements base on their type.
4419 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
4423 // Look for the next elements with the same type.
4424 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
4425 while (SameTypeIt != E &&
4426 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
4427 VisitedInstrs.insert(*SameTypeIt);
4431 // Try to vectorize them.
4432 unsigned NumElts = (SameTypeIt - IncIt);
4433 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n");
4434 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R)) {
4435 // Success start over because instructions might have been changed.
4436 HaveVectorizedPhiNodes = true;
4441 // Start over at the next instruction of a different type (or the end).
4446 VisitedInstrs.clear();
4448 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
4449 // We may go through BB multiple times so skip the one we have checked.
4450 if (!VisitedInstrs.insert(&*it).second)
4453 if (isa<DbgInfoIntrinsic>(it))
4456 // Try to vectorize reductions that use PHINodes.
4457 if (PHINode *P = dyn_cast<PHINode>(it)) {
4458 // Check that the PHI is a reduction PHI.
4459 if (P->getNumIncomingValues() != 2)
4462 Value *Rdx = getReductionValue(DT, P, BB, LI);
4464 // Check if this is a Binary Operator.
4465 BinaryOperator *BI = dyn_cast_or_null<BinaryOperator>(Rdx);
4469 // Try to match and vectorize a horizontal reduction.
4470 if (canMatchHorizontalReduction(P, BI, R, TTI, R.getMinVecRegSize())) {
4477 Value *Inst = BI->getOperand(0);
4479 Inst = BI->getOperand(1);
4481 if (tryToVectorize(dyn_cast<BinaryOperator>(Inst), R)) {
4482 // We would like to start over since some instructions are deleted
4483 // and the iterator may become invalid value.
4493 if (ShouldStartVectorizeHorAtStore)
4494 if (StoreInst *SI = dyn_cast<StoreInst>(it))
4495 if (BinaryOperator *BinOp =
4496 dyn_cast<BinaryOperator>(SI->getValueOperand())) {
4497 if (canMatchHorizontalReduction(nullptr, BinOp, R, TTI,
4498 R.getMinVecRegSize()) ||
4499 tryToVectorize(BinOp, R)) {
4507 // Try to vectorize horizontal reductions feeding into a return.
4508 if (ReturnInst *RI = dyn_cast<ReturnInst>(it))
4509 if (RI->getNumOperands() != 0)
4510 if (BinaryOperator *BinOp =
4511 dyn_cast<BinaryOperator>(RI->getOperand(0))) {
4512 DEBUG(dbgs() << "SLP: Found a return to vectorize.\n");
4513 if (tryToVectorizePair(BinOp->getOperand(0),
4514 BinOp->getOperand(1), R)) {
4522 // Try to vectorize trees that start at compare instructions.
4523 if (CmpInst *CI = dyn_cast<CmpInst>(it)) {
4524 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R)) {
4526 // We would like to start over since some instructions are deleted
4527 // and the iterator may become invalid value.
4533 for (int i = 0; i < 2; ++i) {
4534 if (BinaryOperator *BI = dyn_cast<BinaryOperator>(CI->getOperand(i))) {
4535 if (tryToVectorizePair(BI->getOperand(0), BI->getOperand(1), R)) {
4537 // We would like to start over since some instructions are deleted
4538 // and the iterator may become invalid value.
4548 // Try to vectorize trees that start at insertelement instructions.
4549 if (InsertElementInst *FirstInsertElem = dyn_cast<InsertElementInst>(it)) {
4550 SmallVector<Value *, 16> BuildVector;
4551 SmallVector<Value *, 16> BuildVectorOpds;
4552 if (!findBuildVector(FirstInsertElem, BuildVector, BuildVectorOpds))
4555 // Vectorize starting with the build vector operands ignoring the
4556 // BuildVector instructions for the purpose of scheduling and user
4558 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector)) {
4567 // Try to vectorize trees that start at insertvalue instructions feeding into
4569 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
4570 if (InsertValueInst *LastInsertValue = dyn_cast<InsertValueInst>(SI->getValueOperand())) {
4571 const DataLayout &DL = BB->getModule()->getDataLayout();
4572 if (R.canMapToVector(SI->getValueOperand()->getType(), DL)) {
4573 SmallVector<Value *, 16> BuildVector;
4574 SmallVector<Value *, 16> BuildVectorOpds;
4575 if (!findBuildAggregate(LastInsertValue, BuildVector, BuildVectorOpds))
4578 DEBUG(dbgs() << "SLP: store of array mappable to vector: " << *SI << "\n");
4579 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector, false)) {
4593 bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
4594 auto Changed = false;
4595 for (auto &Entry : GEPs) {
4597 // If the getelementptr list has fewer than two elements, there's nothing
4599 if (Entry.second.size() < 2)
4602 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "
4603 << Entry.second.size() << ".\n");
4605 // We process the getelementptr list in chunks of 16 (like we do for
4606 // stores) to minimize compile-time.
4607 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
4608 auto Len = std::min<unsigned>(BE - BI, 16);
4609 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
4611 // Initialize a set a candidate getelementptrs. Note that we use a
4612 // SetVector here to preserve program order. If the index computations
4613 // are vectorizable and begin with loads, we want to minimize the chance
4614 // of having to reorder them later.
4615 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
4617 // Some of the candidates may have already been vectorized after we
4618 // initially collected them. If so, the WeakVHs will have nullified the
4619 // values, so remove them from the set of candidates.
4620 Candidates.remove(nullptr);
4622 // Remove from the set of candidates all pairs of getelementptrs with
4623 // constant differences. Such getelementptrs are likely not good
4624 // candidates for vectorization in a bottom-up phase since one can be
4625 // computed from the other. We also ensure all candidate getelementptr
4626 // indices are unique.
4627 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
4628 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
4629 if (!Candidates.count(GEPI))
4631 auto *SCEVI = SE->getSCEV(GEPList[I]);
4632 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
4633 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
4634 auto *SCEVJ = SE->getSCEV(GEPList[J]);
4635 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
4636 Candidates.remove(GEPList[I]);
4637 Candidates.remove(GEPList[J]);
4638 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
4639 Candidates.remove(GEPList[J]);
4644 // We break out of the above computation as soon as we know there are
4645 // fewer than two candidates remaining.
4646 if (Candidates.size() < 2)
4649 // Add the single, non-constant index of each candidate to the bundle. We
4650 // ensured the indices met these constraints when we originally collected
4651 // the getelementptrs.
4652 SmallVector<Value *, 16> Bundle(Candidates.size());
4653 auto BundleIndex = 0u;
4654 for (auto *V : Candidates) {
4655 auto *GEP = cast<GetElementPtrInst>(V);
4656 auto *GEPIdx = GEP->idx_begin()->get();
4657 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx));
4658 Bundle[BundleIndex++] = GEPIdx;
4661 // Try and vectorize the indices. We are currently only interested in
4662 // gather-like cases of the form:
4664 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
4666 // where the loads of "a", the loads of "b", and the subtractions can be
4667 // performed in parallel. It's likely that detecting this pattern in a
4668 // bottom-up phase will be simpler and less costly than building a
4669 // full-blown top-down phase beginning at the consecutive loads.
4670 Changed |= tryToVectorizeList(Bundle, R);
4676 bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
4677 bool Changed = false;
4678 // Attempt to sort and vectorize each of the store-groups.
4679 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
4681 if (it->second.size() < 2)
4684 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "
4685 << it->second.size() << ".\n");
4687 // Process the stores in chunks of 16.
4688 // TODO: The limit of 16 inhibits greater vectorization factors.
4689 // For example, AVX2 supports v32i8. Increasing this limit, however,
4690 // may cause a significant compile-time increase.
4691 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
4692 unsigned Len = std::min<unsigned>(CE - CI, 16);
4693 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len),
4694 -SLPCostThreshold, R);
4700 char SLPVectorizer::ID = 0;
4701 static const char lv_name[] = "SLP Vectorizer";
4702 INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)
4703 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
4704 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
4705 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
4706 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
4707 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
4708 INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)
4709 INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)
4712 Pass *createSLPVectorizerPass() { return new SLPVectorizer(); }