1 //===- llvm/Transforms/Utils/LoopUtils.h - Loop utilities -*- C++ -*-=========//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines some loop transformation utilities.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
15 #define LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/Analysis/AliasAnalysis.h"
19 #include "llvm/Analysis/EHPersonalities.h"
20 #include "llvm/IR/Dominators.h"
21 #include "llvm/IR/IRBuilder.h"
25 class AliasSetTracker;
26 class AssumptionCache;
32 class OptimizationRemarkEmitter;
34 class PredicatedScalarEvolution;
35 class PredIteratorCache;
36 class ScalarEvolution;
38 class TargetLibraryInfo;
40 /// \brief Captures loop safety information.
41 /// It keep information for loop & its header may throw exception.
42 struct LoopSafetyInfo {
43 bool MayThrow; // The current loop contains an instruction which
45 bool HeaderMayThrow; // Same as previous, but specific to loop header
46 // Used to update funclet bundle operands.
47 DenseMap<BasicBlock *, ColorVector> BlockColors;
48 LoopSafetyInfo() : MayThrow(false), HeaderMayThrow(false) {}
51 /// The RecurrenceDescriptor is used to identify recurrences variables in a
52 /// loop. Reduction is a special case of recurrence that has uses of the
53 /// recurrence variable outside the loop. The method isReductionPHI identifies
54 /// reductions that are basic recurrences.
56 /// Basic recurrences are defined as the summation, product, OR, AND, XOR, min,
57 /// or max of a set of terms. For example: for(i=0; i<n; i++) { total +=
58 /// array[i]; } is a summation of array elements. Basic recurrences are a
59 /// special case of chains of recurrences (CR). See ScalarEvolution for CR
62 /// This struct holds information about recurrence variables.
63 class RecurrenceDescriptor {
66 /// This enum represents the kinds of recurrences that we support.
68 RK_NoRecurrence, ///< Not a recurrence.
69 RK_IntegerAdd, ///< Sum of integers.
70 RK_IntegerMult, ///< Product of integers.
71 RK_IntegerOr, ///< Bitwise or logical OR of numbers.
72 RK_IntegerAnd, ///< Bitwise or logical AND of numbers.
73 RK_IntegerXor, ///< Bitwise or logical XOR of numbers.
74 RK_IntegerMinMax, ///< Min/max implemented in terms of select(cmp()).
75 RK_FloatAdd, ///< Sum of floats.
76 RK_FloatMult, ///< Product of floats.
77 RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()).
80 // This enum represents the kind of minmax recurrence.
81 enum MinMaxRecurrenceKind {
91 RecurrenceDescriptor()
92 : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoRecurrence),
93 MinMaxKind(MRK_Invalid), UnsafeAlgebraInst(nullptr),
94 RecurrenceType(nullptr), IsSigned(false) {}
96 RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurrenceKind K,
97 MinMaxRecurrenceKind MK, Instruction *UAI, Type *RT,
98 bool Signed, SmallPtrSetImpl<Instruction *> &CI)
99 : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK),
100 UnsafeAlgebraInst(UAI), RecurrenceType(RT), IsSigned(Signed) {
101 CastInsts.insert(CI.begin(), CI.end());
104 /// This POD struct holds information about a potential recurrence operation.
108 InstDesc(bool IsRecur, Instruction *I, Instruction *UAI = nullptr)
109 : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid),
110 UnsafeAlgebraInst(UAI) {}
112 InstDesc(Instruction *I, MinMaxRecurrenceKind K, Instruction *UAI = nullptr)
113 : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K),
114 UnsafeAlgebraInst(UAI) {}
116 bool isRecurrence() { return IsRecurrence; }
118 bool hasUnsafeAlgebra() { return UnsafeAlgebraInst != nullptr; }
120 Instruction *getUnsafeAlgebraInst() { return UnsafeAlgebraInst; }
122 MinMaxRecurrenceKind getMinMaxKind() { return MinMaxKind; }
124 Instruction *getPatternInst() { return PatternLastInst; }
127 // Is this instruction a recurrence candidate.
129 // The last instruction in a min/max pattern (select of the select(icmp())
130 // pattern), or the current recurrence instruction otherwise.
131 Instruction *PatternLastInst;
132 // If this is a min/max pattern the comparison predicate.
133 MinMaxRecurrenceKind MinMaxKind;
134 // Recurrence has unsafe algebra.
135 Instruction *UnsafeAlgebraInst;
138 /// Returns a struct describing if the instruction 'I' can be a recurrence
139 /// variable of type 'Kind'. If the recurrence is a min/max pattern of
140 /// select(icmp()) this function advances the instruction pointer 'I' from the
141 /// compare instruction to the select instruction and stores this pointer in
142 /// 'PatternLastInst' member of the returned struct.
143 static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind,
144 InstDesc &Prev, bool HasFunNoNaNAttr);
146 /// Returns true if instruction I has multiple uses in Insts
147 static bool hasMultipleUsesOf(Instruction *I,
148 SmallPtrSetImpl<Instruction *> &Insts);
150 /// Returns true if all uses of the instruction I is within the Set.
151 static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl<Instruction *> &Set);
153 /// Returns a struct describing if the instruction if the instruction is a
154 /// Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y)
156 static InstDesc isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev);
158 /// Returns identity corresponding to the RecurrenceKind.
159 static Constant *getRecurrenceIdentity(RecurrenceKind K, Type *Tp);
161 /// Returns the opcode of binary operation corresponding to the
163 static unsigned getRecurrenceBinOp(RecurrenceKind Kind);
165 /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
166 static Value *createMinMaxOp(IRBuilder<> &Builder, MinMaxRecurrenceKind RK,
167 Value *Left, Value *Right);
169 /// Returns true if Phi is a reduction of type Kind and adds it to the
170 /// RecurrenceDescriptor.
171 static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop,
172 bool HasFunNoNaNAttr,
173 RecurrenceDescriptor &RedDes);
175 /// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor is
176 /// returned in RedDes.
177 static bool isReductionPHI(PHINode *Phi, Loop *TheLoop,
178 RecurrenceDescriptor &RedDes);
180 /// Returns true if Phi is a first-order recurrence. A first-order recurrence
181 /// is a non-reduction recurrence relation in which the value of the
182 /// recurrence in the current loop iteration equals a value defined in the
183 /// previous iteration.
184 static bool isFirstOrderRecurrence(PHINode *Phi, Loop *TheLoop,
187 RecurrenceKind getRecurrenceKind() { return Kind; }
189 MinMaxRecurrenceKind getMinMaxRecurrenceKind() { return MinMaxKind; }
191 TrackingVH<Value> getRecurrenceStartValue() { return StartValue; }
193 Instruction *getLoopExitInstr() { return LoopExitInstr; }
195 /// Returns true if the recurrence has unsafe algebra which requires a relaxed
196 /// floating-point model.
197 bool hasUnsafeAlgebra() { return UnsafeAlgebraInst != nullptr; }
199 /// Returns first unsafe algebra instruction in the PHI node's use-chain.
200 Instruction *getUnsafeAlgebraInst() { return UnsafeAlgebraInst; }
202 /// Returns true if the recurrence kind is an integer kind.
203 static bool isIntegerRecurrenceKind(RecurrenceKind Kind);
205 /// Returns true if the recurrence kind is a floating point kind.
206 static bool isFloatingPointRecurrenceKind(RecurrenceKind Kind);
208 /// Returns true if the recurrence kind is an arithmetic kind.
209 static bool isArithmeticRecurrenceKind(RecurrenceKind Kind);
211 /// Determines if Phi may have been type-promoted. If Phi has a single user
212 /// that ANDs the Phi with a type mask, return the user. RT is updated to
213 /// account for the narrower bit width represented by the mask, and the AND
214 /// instruction is added to CI.
215 static Instruction *lookThroughAnd(PHINode *Phi, Type *&RT,
216 SmallPtrSetImpl<Instruction *> &Visited,
217 SmallPtrSetImpl<Instruction *> &CI);
219 /// Returns true if all the source operands of a recurrence are either
220 /// SExtInsts or ZExtInsts. This function is intended to be used with
221 /// lookThroughAnd to determine if the recurrence has been type-promoted. The
222 /// source operands are added to CI, and IsSigned is updated to indicate if
223 /// all source operands are SExtInsts.
224 static bool getSourceExtensionKind(Instruction *Start, Instruction *Exit,
225 Type *RT, bool &IsSigned,
226 SmallPtrSetImpl<Instruction *> &Visited,
227 SmallPtrSetImpl<Instruction *> &CI);
229 /// Returns the type of the recurrence. This type can be narrower than the
230 /// actual type of the Phi if the recurrence has been type-promoted.
231 Type *getRecurrenceType() { return RecurrenceType; }
233 /// Returns a reference to the instructions used for type-promoting the
235 SmallPtrSet<Instruction *, 8> &getCastInsts() { return CastInsts; }
237 /// Returns true if all source operands of the recurrence are SExtInsts.
238 bool isSigned() { return IsSigned; }
241 // The starting value of the recurrence.
242 // It does not have to be zero!
243 TrackingVH<Value> StartValue;
244 // The instruction who's value is used outside the loop.
245 Instruction *LoopExitInstr;
246 // The kind of the recurrence.
248 // If this a min/max recurrence the kind of recurrence.
249 MinMaxRecurrenceKind MinMaxKind;
250 // First occurrence of unasfe algebra in the PHI's use-chain.
251 Instruction *UnsafeAlgebraInst;
252 // The type of the recurrence.
253 Type *RecurrenceType;
254 // True if all source operands of the recurrence are SExtInsts.
256 // Instructions used for type-promoting the recurrence.
257 SmallPtrSet<Instruction *, 8> CastInsts;
260 /// A struct for saving information about induction variables.
261 class InductionDescriptor {
263 /// This enum represents the kinds of inductions that we support.
265 IK_NoInduction, ///< Not an induction variable.
266 IK_IntInduction, ///< Integer induction variable. Step = C.
267 IK_PtrInduction, ///< Pointer induction var. Step = C / sizeof(elem).
268 IK_FpInduction ///< Floating point induction variable.
272 /// Default constructor - creates an invalid induction.
273 InductionDescriptor()
274 : StartValue(nullptr), IK(IK_NoInduction), Step(nullptr),
275 InductionBinOp(nullptr) {}
277 /// Get the consecutive direction. Returns:
278 /// 0 - unknown or non-consecutive.
279 /// 1 - consecutive and increasing.
280 /// -1 - consecutive and decreasing.
281 int getConsecutiveDirection() const;
283 /// Compute the transformed value of Index at offset StartValue using step
285 /// For integer induction, returns StartValue + Index * StepValue.
286 /// For pointer induction, returns StartValue[Index * StepValue].
287 /// FIXME: The newly created binary instructions should contain nsw/nuw
288 /// flags, which can be found from the original scalar operations.
289 Value *transform(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
290 const DataLayout& DL) const;
292 Value *getStartValue() const { return StartValue; }
293 InductionKind getKind() const { return IK; }
294 const SCEV *getStep() const { return Step; }
295 ConstantInt *getConstIntStepValue() const;
297 /// Returns true if \p Phi is an induction in the loop \p L. If \p Phi is an
298 /// induction, the induction descriptor \p D will contain the data describing
299 /// this induction. If by some other means the caller has a better SCEV
300 /// expression for \p Phi than the one returned by the ScalarEvolution
301 /// analysis, it can be passed through \p Expr.
302 static bool isInductionPHI(PHINode *Phi, const Loop* L, ScalarEvolution *SE,
303 InductionDescriptor &D,
304 const SCEV *Expr = nullptr);
306 /// Returns true if \p Phi is a floating point induction in the loop \p L.
307 /// If \p Phi is an induction, the induction descriptor \p D will contain
308 /// the data describing this induction.
309 static bool isFPInductionPHI(PHINode *Phi, const Loop* L,
310 ScalarEvolution *SE, InductionDescriptor &D);
312 /// Returns true if \p Phi is a loop \p L induction, in the context associated
313 /// with the run-time predicate of PSE. If \p Assume is true, this can add
314 /// further SCEV predicates to \p PSE in order to prove that \p Phi is an
316 /// If \p Phi is an induction, \p D will contain the data describing this
318 static bool isInductionPHI(PHINode *Phi, const Loop* L,
319 PredicatedScalarEvolution &PSE,
320 InductionDescriptor &D, bool Assume = false);
322 /// Returns true if the induction type is FP and the binary operator does
323 /// not have the "fast-math" property. Such operation requires a relaxed FP
325 bool hasUnsafeAlgebra() {
326 return InductionBinOp &&
327 !cast<FPMathOperator>(InductionBinOp)->hasUnsafeAlgebra();
330 /// Returns induction operator that does not have "fast-math" property
331 /// and requires FP unsafe mode.
332 Instruction *getUnsafeAlgebraInst() {
333 if (!InductionBinOp ||
334 cast<FPMathOperator>(InductionBinOp)->hasUnsafeAlgebra())
336 return InductionBinOp;
339 /// Returns binary opcode of the induction operator.
340 Instruction::BinaryOps getInductionOpcode() const {
341 return InductionBinOp ? InductionBinOp->getOpcode() :
342 Instruction::BinaryOpsEnd;
346 /// Private constructor - used by \c isInductionPHI.
347 InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step,
348 BinaryOperator *InductionBinOp = nullptr);
351 TrackingVH<Value> StartValue;
356 // Instruction that advances induction variable.
357 BinaryOperator *InductionBinOp;
360 BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
363 /// Ensures LCSSA form for every instruction from the Worklist in the scope of
364 /// innermost containing loop.
366 /// For the given instruction which have uses outside of the loop, an LCSSA PHI
367 /// node is inserted and the uses outside the loop are rewritten to use this
370 /// LoopInfo and DominatorTree are required and, since the routine makes no
371 /// changes to CFG, preserved.
373 /// Returns true if any modifications are made.
374 bool formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
375 DominatorTree &DT, LoopInfo &LI);
377 /// \brief Put loop into LCSSA form.
379 /// Looks at all instructions in the loop which have uses outside of the
380 /// current loop. For each, an LCSSA PHI node is inserted and the uses outside
381 /// the loop are rewritten to use this node.
383 /// LoopInfo and DominatorTree are required and preserved.
385 /// If ScalarEvolution is passed in, it will be preserved.
387 /// Returns true if any modifications are made to the loop.
388 bool formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE);
390 /// \brief Put a loop nest into LCSSA form.
392 /// This recursively forms LCSSA for a loop nest.
394 /// LoopInfo and DominatorTree are required and preserved.
396 /// If ScalarEvolution is passed in, it will be preserved.
398 /// Returns true if any modifications are made to the loop.
399 bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI,
400 ScalarEvolution *SE);
402 /// \brief Walk the specified region of the CFG (defined by all blocks
403 /// dominated by the specified block, and that are in the current loop) in
404 /// reverse depth first order w.r.t the DominatorTree. This allows us to visit
405 /// uses before definitions, allowing us to sink a loop body in one pass without
406 /// iteration. Takes DomTreeNode, AliasAnalysis, LoopInfo, DominatorTree,
407 /// DataLayout, TargetLibraryInfo, Loop, AliasSet information for all
408 /// instructions of the loop and loop safety information as
409 /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
410 bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
411 TargetLibraryInfo *, Loop *, AliasSetTracker *,
412 LoopSafetyInfo *, OptimizationRemarkEmitter *ORE);
414 /// \brief Walk the specified region of the CFG (defined by all blocks
415 /// dominated by the specified block, and that are in the current loop) in depth
416 /// first order w.r.t the DominatorTree. This allows us to visit definitions
417 /// before uses, allowing us to hoist a loop body in one pass without iteration.
418 /// Takes DomTreeNode, AliasAnalysis, LoopInfo, DominatorTree, DataLayout,
419 /// TargetLibraryInfo, Loop, AliasSet information for all instructions of the
420 /// loop and loop safety information as arguments. Diagnostics is emitted via \p
421 /// ORE. It returns changed status.
422 bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
423 TargetLibraryInfo *, Loop *, AliasSetTracker *,
424 LoopSafetyInfo *, OptimizationRemarkEmitter *ORE);
426 /// \brief Try to promote memory values to scalars by sinking stores out of
427 /// the loop and moving loads to before the loop. We do this by looping over
428 /// the stores in the loop, looking for stores to Must pointers which are
429 /// loop invariant. It takes AliasSet, Loop exit blocks vector, loop exit blocks
430 /// insertion point vector, PredIteratorCache, LoopInfo, DominatorTree, Loop,
431 /// AliasSet information for all instructions of the loop and loop safety
432 /// information as arguments. Diagnostics is emitted via \p ORE. It returns
434 bool promoteLoopAccessesToScalars(AliasSet &, SmallVectorImpl<BasicBlock *> &,
435 SmallVectorImpl<Instruction *> &,
436 PredIteratorCache &, LoopInfo *,
437 DominatorTree *, const TargetLibraryInfo *,
438 Loop *, AliasSetTracker *, LoopSafetyInfo *,
439 OptimizationRemarkEmitter *);
441 /// \brief Computes safety information for a loop
442 /// checks loop body & header for the possibility of may throw
443 /// exception, it takes LoopSafetyInfo and loop as argument.
444 /// Updates safety information in LoopSafetyInfo argument.
445 void computeLoopSafetyInfo(LoopSafetyInfo *, Loop *);
447 /// Returns true if the instruction in a loop is guaranteed to execute at least
449 bool isGuaranteedToExecute(const Instruction &Inst, const DominatorTree *DT,
451 const LoopSafetyInfo *SafetyInfo);
453 /// \brief Returns the instructions that use values defined in the loop.
454 SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L);
456 /// \brief Find string metadata for loop
458 /// If it has a value (e.g. {"llvm.distribute", 1} return the value as an
459 /// operand or null otherwise. If the string metadata is not found return
460 /// Optional's not-a-value.
461 Optional<const MDOperand *> findStringMetadataForLoop(Loop *TheLoop,
464 /// \brief Set input string into loop metadata by keeping other values intact.
465 void addStringMetadataToLoop(Loop *TheLoop, const char *MDString,
468 /// \brief Get a loop's estimated trip count based on branch weight metadata.
469 /// Returns 0 when the count is estimated to be 0, or None when a meaningful
470 /// estimate can not be made.
471 Optional<unsigned> getLoopEstimatedTripCount(Loop *L);
473 /// Helper to consistently add the set of standard passes to a loop pass's \c
476 /// All loop passes should call this as part of implementing their \c
477 /// getAnalysisUsage.
478 void getLoopAnalysisUsage(AnalysisUsage &AU);
480 /// Returns true if the hoister and sinker can handle this instruction.
481 /// If SafetyInfo is null, we are checking for sinking instructions from
482 /// preheader to loop body (no speculation).
483 /// If SafetyInfo is not null, we are checking for hoisting/sinking
484 /// instructions from loop body to preheader/exit. Check if the instruction
485 /// can execute speculatively.
486 /// If \p ORE is set use it to emit optimization remarks.
487 bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
488 Loop *CurLoop, AliasSetTracker *CurAST,
489 LoopSafetyInfo *SafetyInfo,
490 OptimizationRemarkEmitter *ORE = nullptr);