1 //===- LoopVectorize.h ------------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10 // and generates target-independent LLVM-IR.
11 // The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12 // of instructions in order to estimate the profitability of vectorization.
14 // The loop vectorizer combines consecutive loop iterations into a single
15 // 'wide' iteration. After this transformation the index is incremented
16 // by the SIMD vector width, and not by one.
18 // This pass has three parts:
19 // 1. The main loop pass that drives the different parts.
20 // 2. LoopVectorizationLegality - A unit that checks for the legality
21 // of the vectorization.
22 // 3. InnerLoopVectorizer - A unit that performs the actual
23 // widening of instructions.
24 // 4. LoopVectorizationCostModel - A unit that checks for the profitability
25 // of vectorization. It decides on the optimal vector width, which
26 // can be one, if vectorization is not profitable.
28 // There is a development effort going on to migrate loop vectorizer to the
29 // VPlan infrastructure and to introduce outer loop vectorization support (see
30 // docs/Proposal/VectorizationPlan.rst and
31 // http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32 // purpose, we temporarily introduced the VPlan-native vectorization path: an
33 // alternative vectorization path that is natively implemented on top of the
34 // VPlan infrastructure. See EnableVPlanNativePath for enabling.
36 //===----------------------------------------------------------------------===//
38 // The reduction-variable vectorization is based on the paper:
39 // D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
41 // Variable uniformity checks are inspired by:
42 // Karrenberg, R. and Hack, S. Whole Function Vectorization.
44 // The interleaved access vectorization is based on the paper:
45 // Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
48 // Other ideas/concepts are from:
49 // A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
51 // S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52 // Vectorizing Compilers.
54 //===----------------------------------------------------------------------===//
56 #ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H
57 #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H
59 #include "llvm/Analysis/AliasAnalysis.h"
60 #include "llvm/IR/PassManager.h"
61 #include "llvm/Support/CommandLine.h"
66 class AssumptionCache;
67 class BlockFrequencyInfo;
74 class OptimizationRemarkEmitter;
75 class ProfileSummaryInfo;
76 class ScalarEvolution;
77 class TargetLibraryInfo;
78 class TargetTransformInfo;
80 extern cl::opt<bool> EnableLoopInterleaving;
81 extern cl::opt<bool> EnableLoopVectorization;
83 struct LoopVectorizeOptions {
84 /// If false, consider all loops for interleaving.
85 /// If true, only loops that explicitly request interleaving are considered.
86 bool InterleaveOnlyWhenForced;
88 /// If false, consider all loops for vectorization.
89 /// If true, only loops that explicitly request vectorization are considered.
90 bool VectorizeOnlyWhenForced;
92 /// The current defaults when creating the pass with no arguments are:
93 /// EnableLoopInterleaving = true and EnableLoopVectorization = true. This
94 /// means that interleaving default is consistent with the cl::opt flag, while
95 /// vectorization is not.
96 /// FIXME: The default for EnableLoopVectorization in the cl::opt should be
97 /// set to true, and the corresponding change to account for this be made in
98 /// opt.cpp. The initializations below will become:
99 /// InterleaveOnlyWhenForced(!EnableLoopInterleaving)
100 /// VectorizeOnlyWhenForced(!EnableLoopVectorization).
101 LoopVectorizeOptions()
102 : InterleaveOnlyWhenForced(false), VectorizeOnlyWhenForced(false) {}
103 LoopVectorizeOptions(bool InterleaveOnlyWhenForced,
104 bool VectorizeOnlyWhenForced)
105 : InterleaveOnlyWhenForced(InterleaveOnlyWhenForced),
106 VectorizeOnlyWhenForced(VectorizeOnlyWhenForced) {}
108 LoopVectorizeOptions &setInterleaveOnlyWhenForced(bool Value) {
109 InterleaveOnlyWhenForced = Value;
113 LoopVectorizeOptions &setVectorizeOnlyWhenForced(bool Value) {
114 VectorizeOnlyWhenForced = Value;
119 /// The LoopVectorize Pass.
120 struct LoopVectorizePass : public PassInfoMixin<LoopVectorizePass> {
121 /// If false, consider all loops for interleaving.
122 /// If true, only loops that explicitly request interleaving are considered.
123 bool InterleaveOnlyWhenForced;
125 /// If false, consider all loops for vectorization.
126 /// If true, only loops that explicitly request vectorization are considered.
127 bool VectorizeOnlyWhenForced;
129 LoopVectorizePass(LoopVectorizeOptions Opts = {})
130 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced),
131 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {}
135 TargetTransformInfo *TTI;
137 BlockFrequencyInfo *BFI;
138 TargetLibraryInfo *TLI;
142 std::function<const LoopAccessInfo &(Loop &)> *GetLAA;
143 OptimizationRemarkEmitter *ORE;
144 ProfileSummaryInfo *PSI;
146 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
149 bool runImpl(Function &F, ScalarEvolution &SE_, LoopInfo &LI_,
150 TargetTransformInfo &TTI_, DominatorTree &DT_,
151 BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_,
152 DemandedBits &DB_, AliasAnalysis &AA_, AssumptionCache &AC_,
153 std::function<const LoopAccessInfo &(Loop &)> &GetLAA_,
154 OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_);
156 bool processLoop(Loop *L);
159 /// Reports a vectorization failure: print \p DebugMsg for debugging
160 /// purposes along with the corresponding optimization remark \p RemarkName.
161 /// If \p I is passed, it is an instruction that prevents vectorization.
162 /// Otherwise, the loop \p TheLoop is used for the location of the remark.
163 void reportVectorizationFailure(const StringRef DebugMsg,
164 const StringRef OREMsg, const StringRef ORETag,
165 OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I = nullptr);
167 } // end namespace llvm
169 #endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZE_H