1 //===---- NewGVN.cpp - Global Value Numbering Pass --------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// This file implements the new LLVM's Global Value Numbering pass.
11 /// GVN partitions values computed by a function into congruence classes.
12 /// Values ending up in the same congruence class are guaranteed to be the same
13 /// for every execution of the program. In that respect, congruency is a
14 /// compile-time approximation of equivalence of values at runtime.
15 /// The algorithm implemented here uses a sparse formulation and it's based
16 /// on the ideas described in the paper:
17 /// "A Sparse Algorithm for Predicated Global Value Numbering" from
20 /// A brief overview of the algorithm: The algorithm is essentially the same as
21 /// the standard RPO value numbering algorithm (a good reference is the paper
22 /// "SCC based value numbering" by L. Taylor Simpson) with one major difference:
23 /// The RPO algorithm proceeds, on every iteration, to process every reachable
24 /// block and every instruction in that block. This is because the standard RPO
25 /// algorithm does not track what things have the same value number, it only
26 /// tracks what the value number of a given operation is (the mapping is
27 /// operation -> value number). Thus, when a value number of an operation
28 /// changes, it must reprocess everything to ensure all uses of a value number
29 /// get updated properly. In constrast, the sparse algorithm we use *also*
30 /// tracks what operations have a given value number (IE it also tracks the
31 /// reverse mapping from value number -> operations with that value number), so
32 /// that it only needs to reprocess the instructions that are affected when
33 /// something's value number changes. The vast majority of complexity and code
34 /// in this file is devoted to tracking what value numbers could change for what
35 /// instructions when various things happen. The rest of the algorithm is
36 /// devoted to performing symbolic evaluation, forward propagation, and
37 /// simplification of operations based on the value numbers deduced so far
39 /// In order to make the GVN mostly-complete, we use a technique derived from
40 /// "Detection of Redundant Expressions: A Complete and Polynomial-time
41 /// Algorithm in SSA" by R.R. Pai. The source of incompleteness in most SSA
42 /// based GVN algorithms is related to their inability to detect equivalence
43 /// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)).
44 /// We resolve this issue by generating the equivalent "phi of ops" form for
45 /// each op of phis we see, in a way that only takes polynomial time to resolve.
47 /// We also do not perform elimination by using any published algorithm. All
48 /// published algorithms are O(Instructions). Instead, we use a technique that
49 /// is O(number of operations with the same value number), enabling us to skip
50 /// trying to eliminate things that have unique value numbers.
51 //===----------------------------------------------------------------------===//
53 #include "llvm/Transforms/Scalar/NewGVN.h"
54 #include "llvm/ADT/BitVector.h"
55 #include "llvm/ADT/DenseMap.h"
56 #include "llvm/ADT/DenseSet.h"
57 #include "llvm/ADT/DepthFirstIterator.h"
58 #include "llvm/ADT/Hashing.h"
59 #include "llvm/ADT/MapVector.h"
60 #include "llvm/ADT/PostOrderIterator.h"
61 #include "llvm/ADT/STLExtras.h"
62 #include "llvm/ADT/SmallPtrSet.h"
63 #include "llvm/ADT/SmallSet.h"
64 #include "llvm/ADT/Statistic.h"
65 #include "llvm/ADT/TinyPtrVector.h"
66 #include "llvm/Analysis/AliasAnalysis.h"
67 #include "llvm/Analysis/AssumptionCache.h"
68 #include "llvm/Analysis/CFG.h"
69 #include "llvm/Analysis/CFGPrinter.h"
70 #include "llvm/Analysis/ConstantFolding.h"
71 #include "llvm/Analysis/GlobalsModRef.h"
72 #include "llvm/Analysis/InstructionSimplify.h"
73 #include "llvm/Analysis/MemoryBuiltins.h"
74 #include "llvm/Analysis/MemoryLocation.h"
75 #include "llvm/Analysis/MemorySSA.h"
76 #include "llvm/Analysis/TargetLibraryInfo.h"
77 #include "llvm/IR/DataLayout.h"
78 #include "llvm/IR/Dominators.h"
79 #include "llvm/IR/GlobalVariable.h"
80 #include "llvm/IR/IRBuilder.h"
81 #include "llvm/IR/IntrinsicInst.h"
82 #include "llvm/IR/LLVMContext.h"
83 #include "llvm/IR/Metadata.h"
84 #include "llvm/IR/PatternMatch.h"
85 #include "llvm/IR/Type.h"
86 #include "llvm/Support/Allocator.h"
87 #include "llvm/Support/CommandLine.h"
88 #include "llvm/Support/Debug.h"
89 #include "llvm/Support/DebugCounter.h"
90 #include "llvm/Transforms/Scalar.h"
91 #include "llvm/Transforms/Scalar/GVNExpression.h"
92 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
93 #include "llvm/Transforms/Utils/Local.h"
94 #include "llvm/Transforms/Utils/PredicateInfo.h"
95 #include "llvm/Transforms/Utils/VNCoercion.h"
97 #include <unordered_map>
100 using namespace llvm;
101 using namespace PatternMatch;
102 using namespace llvm::GVNExpression;
103 using namespace llvm::VNCoercion;
104 #define DEBUG_TYPE "newgvn"
106 STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
107 STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
108 STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
109 STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
110 STATISTIC(NumGVNMaxIterations,
111 "Maximum Number of iterations it took to converge GVN");
112 STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
113 STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
114 STATISTIC(NumGVNAvoidedSortedLeaderChanges,
115 "Number of avoided sorted leader changes");
116 STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");
117 STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created");
118 STATISTIC(NumGVNPHIOfOpsEliminations,
119 "Number of things eliminated using PHI of ops");
120 DEBUG_COUNTER(VNCounter, "newgvn-vn",
121 "Controls which instructions are value numbered")
122 DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",
123 "Controls which instructions we create phi of ops for")
124 // Currently store defining access refinement is too slow due to basicaa being
125 // egregiously slow. This flag lets us keep it working while we work on this
127 static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
128 cl::init(false), cl::Hidden);
130 //===----------------------------------------------------------------------===//
132 //===----------------------------------------------------------------------===//
136 namespace GVNExpression {
137 Expression::~Expression() = default;
138 BasicExpression::~BasicExpression() = default;
139 CallExpression::~CallExpression() = default;
140 LoadExpression::~LoadExpression() = default;
141 StoreExpression::~StoreExpression() = default;
142 AggregateValueExpression::~AggregateValueExpression() = default;
143 PHIExpression::~PHIExpression() = default;
147 // Tarjan's SCC finding algorithm with Nuutila's improvements
148 // SCCIterator is actually fairly complex for the simple thing we want.
149 // It also wants to hand us SCC's that are unrelated to the phi node we ask
150 // about, and have us process them there or risk redoing work.
151 // Graph traits over a filter iterator also doesn't work that well here.
152 // This SCC finder is specialized to walk use-def chains, and only follows
154 // not generic values (arguments, etc).
157 TarjanSCC() : Components(1) {}
159 void Start(const Instruction *Start) {
160 if (Root.lookup(Start) == 0)
164 const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {
165 unsigned ComponentID = ValueToComponent.lookup(V);
167 assert(ComponentID > 0 &&
168 "Asking for a component for a value we never processed");
169 return Components[ComponentID];
173 void FindSCC(const Instruction *I) {
175 // Store the DFS Number we had before it possibly gets incremented.
176 unsigned int OurDFS = DFSNum;
177 for (auto &Op : I->operands()) {
178 if (auto *InstOp = dyn_cast<Instruction>(Op)) {
179 if (Root.lookup(Op) == 0)
181 if (!InComponent.count(Op))
182 Root[I] = std::min(Root.lookup(I), Root.lookup(Op));
185 // See if we really were the root of a component, by seeing if we still have
186 // our DFSNumber. If we do, we are the root of the component, and we have
187 // completed a component. If we do not, we are not the root of a component,
188 // and belong on the component stack.
189 if (Root.lookup(I) == OurDFS) {
190 unsigned ComponentID = Components.size();
191 Components.resize(Components.size() + 1);
192 auto &Component = Components.back();
194 DEBUG(dbgs() << "Component root is " << *I << "\n");
195 InComponent.insert(I);
196 ValueToComponent[I] = ComponentID;
197 // Pop a component off the stack and label it.
198 while (!Stack.empty() && Root.lookup(Stack.back()) >= OurDFS) {
199 auto *Member = Stack.back();
200 DEBUG(dbgs() << "Component member is " << *Member << "\n");
201 Component.insert(Member);
202 InComponent.insert(Member);
203 ValueToComponent[Member] = ComponentID;
207 // Part of a component, push to stack
211 unsigned int DFSNum = 1;
212 SmallPtrSet<const Value *, 8> InComponent;
213 DenseMap<const Value *, unsigned int> Root;
214 SmallVector<const Value *, 8> Stack;
215 // Store the components as vector of ptr sets, because we need the topo order
216 // of SCC's, but not individual member order
217 SmallVector<SmallPtrSet<const Value *, 8>, 8> Components;
218 DenseMap<const Value *, unsigned> ValueToComponent;
220 // Congruence classes represent the set of expressions/instructions
221 // that are all the same *during some scope in the function*.
222 // That is, because of the way we perform equality propagation, and
223 // because of memory value numbering, it is not correct to assume
224 // you can willy-nilly replace any member with any other at any
225 // point in the function.
227 // For any Value in the Member set, it is valid to replace any dominated member
230 // Every congruence class has a leader, and the leader is used to symbolize
231 // instructions in a canonical way (IE every operand of an instruction that is a
232 // member of the same congruence class will always be replaced with leader
233 // during symbolization). To simplify symbolization, we keep the leader as a
234 // constant if class can be proved to be a constant value. Otherwise, the
235 // leader is the member of the value set with the smallest DFS number. Each
236 // congruence class also has a defining expression, though the expression may be
237 // null. If it exists, it can be used for forward propagation and reassociation
240 // For memory, we also track a representative MemoryAccess, and a set of memory
241 // members for MemoryPhis (which have no real instructions). Note that for
242 // memory, it seems tempting to try to split the memory members into a
243 // MemoryCongruenceClass or something. Unfortunately, this does not work
244 // easily. The value numbering of a given memory expression depends on the
245 // leader of the memory congruence class, and the leader of memory congruence
246 // class depends on the value numbering of a given memory expression. This
247 // leads to wasted propagation, and in some cases, missed optimization. For
248 // example: If we had value numbered two stores together before, but now do not,
249 // we move them to a new value congruence class. This in turn will move at one
250 // of the memorydefs to a new memory congruence class. Which in turn, affects
251 // the value numbering of the stores we just value numbered (because the memory
252 // congruence class is part of the value number). So while theoretically
253 // possible to split them up, it turns out to be *incredibly* complicated to get
254 // it to work right, because of the interdependency. While structurally
255 // slightly messier, it is algorithmically much simpler and faster to do what we
256 // do here, and track them both at once in the same class.
257 // Note: The default iterators for this class iterate over values
258 class CongruenceClass {
260 using MemberType = Value;
261 using MemberSet = SmallPtrSet<MemberType *, 4>;
262 using MemoryMemberType = MemoryPhi;
263 using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;
265 explicit CongruenceClass(unsigned ID) : ID(ID) {}
266 CongruenceClass(unsigned ID, Value *Leader, const Expression *E)
267 : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
268 unsigned getID() const { return ID; }
269 // True if this class has no members left. This is mainly used for assertion
270 // purposes, and for skipping empty classes.
271 bool isDead() const {
272 // If it's both dead from a value perspective, and dead from a memory
273 // perspective, it's really dead.
274 return empty() && memory_empty();
277 Value *getLeader() const { return RepLeader; }
278 void setLeader(Value *Leader) { RepLeader = Leader; }
279 const std::pair<Value *, unsigned int> &getNextLeader() const {
282 void resetNextLeader() { NextLeader = {nullptr, ~0}; }
284 void addPossibleNextLeader(std::pair<Value *, unsigned int> LeaderPair) {
285 if (LeaderPair.second < NextLeader.second)
286 NextLeader = LeaderPair;
289 Value *getStoredValue() const { return RepStoredValue; }
290 void setStoredValue(Value *Leader) { RepStoredValue = Leader; }
291 const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }
292 void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }
294 // Forward propagation info
295 const Expression *getDefiningExpr() const { return DefiningExpr; }
298 bool empty() const { return Members.empty(); }
299 unsigned size() const { return Members.size(); }
300 MemberSet::const_iterator begin() const { return Members.begin(); }
301 MemberSet::const_iterator end() const { return Members.end(); }
302 void insert(MemberType *M) { Members.insert(M); }
303 void erase(MemberType *M) { Members.erase(M); }
304 void swap(MemberSet &Other) { Members.swap(Other); }
307 bool memory_empty() const { return MemoryMembers.empty(); }
308 unsigned memory_size() const { return MemoryMembers.size(); }
309 MemoryMemberSet::const_iterator memory_begin() const {
310 return MemoryMembers.begin();
312 MemoryMemberSet::const_iterator memory_end() const {
313 return MemoryMembers.end();
315 iterator_range<MemoryMemberSet::const_iterator> memory() const {
316 return make_range(memory_begin(), memory_end());
318 void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); }
319 void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); }
322 unsigned getStoreCount() const { return StoreCount; }
323 void incStoreCount() { ++StoreCount; }
324 void decStoreCount() {
325 assert(StoreCount != 0 && "Store count went negative");
329 // True if this class has no memory members.
330 bool definesNoMemory() const { return StoreCount == 0 && memory_empty(); }
332 // Return true if two congruence classes are equivalent to each other. This
334 // that every field but the ID number and the dead field are equivalent.
335 bool isEquivalentTo(const CongruenceClass *Other) const {
341 if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) !=
342 std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue,
343 Other->RepMemoryAccess))
345 if (DefiningExpr != Other->DefiningExpr)
346 if (!DefiningExpr || !Other->DefiningExpr ||
347 *DefiningExpr != *Other->DefiningExpr)
349 // We need some ordered set
350 std::set<Value *> AMembers(Members.begin(), Members.end());
351 std::set<Value *> BMembers(Members.begin(), Members.end());
352 return AMembers == BMembers;
357 // Representative leader.
358 Value *RepLeader = nullptr;
359 // The most dominating leader after our current leader, because the member set
360 // is not sorted and is expensive to keep sorted all the time.
361 std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
362 // If this is represented by a store, the value of the store.
363 Value *RepStoredValue = nullptr;
364 // If this class contains MemoryDefs or MemoryPhis, this is the leading memory
366 const MemoryAccess *RepMemoryAccess = nullptr;
367 // Defining Expression.
368 const Expression *DefiningExpr = nullptr;
369 // Actual members of this class.
371 // This is the set of MemoryPhis that exist in the class. MemoryDefs and
372 // MemoryUses have real instructions representing them, so we only need to
373 // track MemoryPhis here.
374 MemoryMemberSet MemoryMembers;
375 // Number of stores in this congruence class.
376 // This is used so we can detect store equivalence changes properly.
380 struct HashedExpression;
382 template <> struct DenseMapInfo<const Expression *> {
383 static const Expression *getEmptyKey() {
384 auto Val = static_cast<uintptr_t>(-1);
385 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
386 return reinterpret_cast<const Expression *>(Val);
388 static const Expression *getTombstoneKey() {
389 auto Val = static_cast<uintptr_t>(~1U);
390 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
391 return reinterpret_cast<const Expression *>(Val);
393 static unsigned getHashValue(const Expression *E) {
394 return static_cast<unsigned>(E->getHashValue());
396 static unsigned getHashValue(const HashedExpression &HE);
397 static bool isEqual(const HashedExpression &LHS, const Expression *RHS);
398 static bool isEqual(const Expression *LHS, const Expression *RHS) {
401 if (LHS == getTombstoneKey() || RHS == getTombstoneKey() ||
402 LHS == getEmptyKey() || RHS == getEmptyKey())
407 } // end namespace llvm
409 // This is just a wrapper around Expression that computes the hash value once at
410 // creation time. Hash values for an Expression can't change once they are
411 // inserted into the DenseMap (it breaks DenseMap), so they must be immutable at
412 // that point anyway.
413 struct HashedExpression {
416 HashedExpression(const Expression *E)
417 : E(E), HashVal(DenseMapInfo<const Expression *>::getHashValue(E)) {}
421 DenseMapInfo<const Expression *>::getHashValue(const HashedExpression &HE) {
424 bool DenseMapInfo<const Expression *>::isEqual(const HashedExpression &LHS,
425 const Expression *RHS) {
426 return isEqual(LHS.E, RHS);
433 const TargetLibraryInfo *TLI;
436 MemorySSAWalker *MSSAWalker;
437 const DataLayout &DL;
438 std::unique_ptr<PredicateInfo> PredInfo;
440 // These are the only two things the create* functions should have
441 // side-effects on due to allocating memory.
442 mutable BumpPtrAllocator ExpressionAllocator;
443 mutable ArrayRecycler<Value *> ArgRecycler;
444 mutable TarjanSCC SCCFinder;
445 const SimplifyQuery SQ;
447 // Number of function arguments, used by ranking
448 unsigned int NumFuncArgs;
450 // RPOOrdering of basic blocks
451 DenseMap<const DomTreeNode *, unsigned> RPOOrdering;
453 // Congruence class info.
455 // This class is called INITIAL in the paper. It is the class everything
456 // startsout in, and represents any value. Being an optimistic analysis,
457 // anything in the TOP class has the value TOP, which is indeterminate and
458 // equivalent to everything.
459 CongruenceClass *TOPClass;
460 std::vector<CongruenceClass *> CongruenceClasses;
461 unsigned NextCongruenceNum;
464 DenseMap<Value *, CongruenceClass *> ValueToClass;
465 DenseMap<Value *, const Expression *> ValueToExpression;
466 // Value PHI handling, used to make equivalence between phi(op, op) and
468 // These mappings just store various data that would normally be part of the
470 DenseSet<const Instruction *> PHINodeUses;
471 // Map a temporary instruction we created to a parent block.
472 DenseMap<const Value *, BasicBlock *> TempToBlock;
473 // Map between the temporary phis we created and the real instructions they
474 // are known equivalent to.
475 DenseMap<const Value *, PHINode *> RealToTemp;
476 // In order to know when we should re-process instructions that have
477 // phi-of-ops, we track the set of expressions that they needed as
478 // leaders. When we discover new leaders for those expressions, we process the
479 // associated phi-of-op instructions again in case they have changed. The
480 // other way they may change is if they had leaders, and those leaders
481 // disappear. However, at the point they have leaders, there are uses of the
482 // relevant operands in the created phi node, and so they will get reprocessed
483 // through the normal user marking we perform.
484 mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers;
485 DenseMap<const Expression *, SmallPtrSet<Instruction *, 2>>
486 ExpressionToPhiOfOps;
487 // Map from basic block to the temporary operations we created
488 DenseMap<const BasicBlock *, SmallVector<PHINode *, 8>> PHIOfOpsPHIs;
489 // Map from temporary operation to MemoryAccess.
490 DenseMap<const Instruction *, MemoryUseOrDef *> TempToMemory;
491 // Set of all temporary instructions we created.
492 DenseSet<Instruction *> AllTempInstructions;
494 // Mapping from predicate info we used to the instructions we used it with.
495 // In order to correctly ensure propagation, we must keep track of what
496 // comparisons we used, so that when the values of the comparisons change, we
497 // propagate the information to the places we used the comparison.
498 mutable DenseMap<const Value *, SmallPtrSet<Instruction *, 2>>
500 // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for
501 // stores, we no longer can rely solely on the def-use chains of MemorySSA.
502 mutable DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>>
505 // A table storing which memorydefs/phis represent a memory state provably
506 // equivalent to another memory state.
507 // We could use the congruence class machinery, but the MemoryAccess's are
508 // abstract memory states, so they can only ever be equivalent to each other,
509 // and not to constants, etc.
510 DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass;
512 // We could, if we wanted, build MemoryPhiExpressions and
513 // MemoryVariableExpressions, etc, and value number them the same way we value
514 // number phi expressions. For the moment, this seems like overkill. They
515 // can only exist in one of three states: they can be TOP (equal to
516 // everything), Equivalent to something else, or unique. Because we do not
517 // create expressions for them, we need to simulate leader change not just
518 // when they change class, but when they change state. Note: We can do the
519 // same thing for phis, and avoid having phi expressions if we wanted, We
520 // should eventually unify in one direction or the other, so this is a little
521 // bit of an experiment in which turns out easier to maintain.
522 enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };
523 DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState;
525 enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle };
526 mutable DenseMap<const Instruction *, InstCycleState> InstCycleState;
527 // Expression to class mapping.
528 using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
529 ExpressionClassMap ExpressionToClass;
531 // We have a single expression that represents currently DeadExpressions.
532 // For dead expressions we can prove will stay dead, we mark them with
533 // DFS number zero. However, it's possible in the case of phi nodes
534 // for us to assume/prove all arguments are dead during fixpointing.
535 // We use DeadExpression for that case.
536 DeadExpression *SingletonDeadExpression = nullptr;
538 // Which values have changed as a result of leader changes.
539 SmallPtrSet<Value *, 8> LeaderChanges;
541 // Reachability info.
542 using BlockEdge = BasicBlockEdge;
543 DenseSet<BlockEdge> ReachableEdges;
544 SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
546 // This is a bitvector because, on larger functions, we may have
547 // thousands of touched instructions at once (entire blocks,
548 // instructions with hundreds of uses, etc). Even with optimization
549 // for when we mark whole blocks as touched, when this was a
550 // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
551 // the time in GVN just managing this list. The bitvector, on the
552 // other hand, efficiently supports test/set/clear of both
553 // individual and ranges, as well as "find next element" This
554 // enables us to use it as a worklist with essentially 0 cost.
555 BitVector TouchedInstructions;
557 DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
560 // Debugging for how many times each block and instruction got processed.
561 DenseMap<const Value *, unsigned> ProcessedCount;
565 // This contains a mapping from Instructions to DFS numbers.
566 // The numbering starts at 1. An instruction with DFS number zero
567 // means that the instruction is dead.
568 DenseMap<const Value *, unsigned> InstrDFS;
570 // This contains the mapping DFS numbers to instructions.
571 SmallVector<Value *, 32> DFSToInstr;
574 SmallPtrSet<Instruction *, 8> InstructionsToErase;
577 NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
578 TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
579 const DataLayout &DL)
580 : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), DL(DL),
581 PredInfo(make_unique<PredicateInfo>(F, *DT, *AC)), SQ(DL, TLI, DT, AC) {
586 // Expression handling.
587 const Expression *createExpression(Instruction *) const;
588 const Expression *createBinaryExpression(unsigned, Type *, Value *,
590 PHIExpression *createPHIExpression(Instruction *, bool &HasBackEdge,
591 bool &OriginalOpsConstant) const;
592 const DeadExpression *createDeadExpression() const;
593 const VariableExpression *createVariableExpression(Value *) const;
594 const ConstantExpression *createConstantExpression(Constant *) const;
595 const Expression *createVariableOrConstant(Value *V) const;
596 const UnknownExpression *createUnknownExpression(Instruction *) const;
597 const StoreExpression *createStoreExpression(StoreInst *,
598 const MemoryAccess *) const;
599 LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
600 const MemoryAccess *) const;
601 const CallExpression *createCallExpression(CallInst *,
602 const MemoryAccess *) const;
603 const AggregateValueExpression *
604 createAggregateValueExpression(Instruction *) const;
605 bool setBasicExpressionInfo(Instruction *, BasicExpression *) const;
607 // Congruence class handling.
608 CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
609 auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E);
610 CongruenceClasses.emplace_back(result);
614 CongruenceClass *createMemoryClass(MemoryAccess *MA) {
615 auto *CC = createCongruenceClass(nullptr, nullptr);
616 CC->setMemoryLeader(MA);
619 CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {
620 auto *CC = getMemoryClass(MA);
621 if (CC->getMemoryLeader() != MA)
622 CC = createMemoryClass(MA);
626 CongruenceClass *createSingletonCongruenceClass(Value *Member) {
627 CongruenceClass *CClass = createCongruenceClass(Member, nullptr);
628 CClass->insert(Member);
629 ValueToClass[Member] = CClass;
632 void initializeCongruenceClasses(Function &F);
633 const Expression *makePossiblePhiOfOps(Instruction *, bool,
634 SmallPtrSetImpl<Value *> &);
635 void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue);
637 // Value number an Instruction or MemoryPhi.
638 void valueNumberMemoryPhi(MemoryPhi *);
639 void valueNumberInstruction(Instruction *);
641 // Symbolic evaluation.
642 const Expression *checkSimplificationResults(Expression *, Instruction *,
644 const Expression *performSymbolicEvaluation(Value *,
645 SmallPtrSetImpl<Value *> &) const;
646 const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
648 MemoryAccess *) const;
649 const Expression *performSymbolicLoadEvaluation(Instruction *) const;
650 const Expression *performSymbolicStoreEvaluation(Instruction *) const;
651 const Expression *performSymbolicCallEvaluation(Instruction *) const;
652 const Expression *performSymbolicPHIEvaluation(Instruction *) const;
653 const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
654 const Expression *performSymbolicCmpEvaluation(Instruction *) const;
655 const Expression *performSymbolicPredicateInfoEvaluation(Instruction *) const;
657 // Congruence finding.
658 bool someEquivalentDominates(const Instruction *, const Instruction *) const;
659 Value *lookupOperandLeader(Value *) const;
660 void performCongruenceFinding(Instruction *, const Expression *);
661 void moveValueToNewCongruenceClass(Instruction *, const Expression *,
662 CongruenceClass *, CongruenceClass *);
663 void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,
664 CongruenceClass *, CongruenceClass *);
665 Value *getNextValueLeader(CongruenceClass *) const;
666 const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;
667 bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);
668 CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;
669 const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;
670 bool isMemoryAccessTOP(const MemoryAccess *) const;
673 unsigned int getRank(const Value *) const;
674 bool shouldSwapOperands(const Value *, const Value *) const;
676 // Reachability handling.
677 void updateReachableEdge(BasicBlock *, BasicBlock *);
678 void processOutgoingEdges(TerminatorInst *, BasicBlock *);
679 Value *findConditionEquivalence(Value *) const;
683 void convertClassToDFSOrdered(const CongruenceClass &,
684 SmallVectorImpl<ValueDFS> &,
685 DenseMap<const Value *, unsigned int> &,
686 SmallPtrSetImpl<Instruction *> &) const;
687 void convertClassToLoadsAndStores(const CongruenceClass &,
688 SmallVectorImpl<ValueDFS> &) const;
690 bool eliminateInstructions(Function &);
691 void replaceInstruction(Instruction *, Value *);
692 void markInstructionForDeletion(Instruction *);
693 void deleteInstructionsInBlock(BasicBlock *);
694 Value *findPhiOfOpsLeader(const Expression *E, const BasicBlock *BB) const;
696 // New instruction creation.
697 void handleNewInstruction(Instruction *){};
699 // Various instruction touch utilities
700 template <typename Map, typename KeyType, typename Func>
701 void for_each_found(Map &, const KeyType &, Func);
702 template <typename Map, typename KeyType>
703 void touchAndErase(Map &, const KeyType &);
704 void markUsersTouched(Value *);
705 void markMemoryUsersTouched(const MemoryAccess *);
706 void markMemoryDefTouched(const MemoryAccess *);
707 void markPredicateUsersTouched(Instruction *);
708 void markValueLeaderChangeTouched(CongruenceClass *CC);
709 void markMemoryLeaderChangeTouched(CongruenceClass *CC);
710 void markPhiOfOpsChanged(const HashedExpression &HE);
711 void addPredicateUsers(const PredicateBase *, Instruction *) const;
712 void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;
713 void addAdditionalUsers(Value *To, Value *User) const;
715 // Main loop of value numbering
716 void iterateTouchedInstructions();
719 void cleanupTables();
720 std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
721 void updateProcessedCount(const Value *V);
722 void verifyMemoryCongruency() const;
723 void verifyIterationSettled(Function &F);
724 void verifyStoreExpressions() const;
725 bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &,
726 const MemoryAccess *, const MemoryAccess *) const;
727 BasicBlock *getBlockForValue(Value *V) const;
728 void deleteExpression(const Expression *E) const;
729 MemoryUseOrDef *getMemoryAccess(const Instruction *) const;
730 MemoryAccess *getDefiningAccess(const MemoryAccess *) const;
731 MemoryPhi *getMemoryAccess(const BasicBlock *) const;
732 template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
733 unsigned InstrToDFSNum(const Value *V) const {
734 assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");
735 return InstrDFS.lookup(V);
738 unsigned InstrToDFSNum(const MemoryAccess *MA) const {
739 return MemoryToDFSNum(MA);
741 Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }
742 // Given a MemoryAccess, return the relevant instruction DFS number. Note:
743 // This deliberately takes a value so it can be used with Use's, which will
744 // auto-convert to Value's but not to MemoryAccess's.
745 unsigned MemoryToDFSNum(const Value *MA) const {
746 assert(isa<MemoryAccess>(MA) &&
747 "This should not be used with instructions");
748 return isa<MemoryUseOrDef>(MA)
749 ? InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst())
750 : InstrDFS.lookup(MA);
752 bool isCycleFree(const Instruction *) const;
753 bool isBackedge(BasicBlock *From, BasicBlock *To) const;
754 // Debug counter info. When verifying, we have to reset the value numbering
755 // debug counter to the same state it started in to get the same results.
756 std::pair<int, int> StartingVNCounter;
758 } // end anonymous namespace
760 template <typename T>
761 static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
762 if (!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS))
764 return LHS.MemoryExpression::equals(RHS);
767 bool LoadExpression::equals(const Expression &Other) const {
768 return equalsLoadStoreHelper(*this, Other);
771 bool StoreExpression::equals(const Expression &Other) const {
772 if (!equalsLoadStoreHelper(*this, Other))
774 // Make sure that store vs store includes the value operand.
775 if (const auto *S = dyn_cast<StoreExpression>(&Other))
776 if (getStoredValue() != S->getStoredValue())
781 // Determine if the edge From->To is a backedge
782 bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const {
785 auto *FromDTN = DT->getNode(From);
786 auto *ToDTN = DT->getNode(To);
787 return RPOOrdering.lookup(FromDTN) >= RPOOrdering.lookup(ToDTN);
791 static std::string getBlockName(const BasicBlock *B) {
792 return DOTGraphTraits<const Function *>::getSimpleNodeLabel(B, nullptr);
796 // Get a MemoryAccess for an instruction, fake or real.
797 MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const {
798 auto *Result = MSSA->getMemoryAccess(I);
799 return Result ? Result : TempToMemory.lookup(I);
802 // Get a MemoryPhi for a basic block. These are all real.
803 MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const {
804 return MSSA->getMemoryAccess(BB);
807 // Get the basic block from an instruction/memory value.
808 BasicBlock *NewGVN::getBlockForValue(Value *V) const {
809 if (auto *I = dyn_cast<Instruction>(V)) {
810 auto *Parent = I->getParent();
813 Parent = TempToBlock.lookup(V);
814 assert(Parent && "Every fake instruction should have a block");
818 auto *MP = dyn_cast<MemoryPhi>(V);
819 assert(MP && "Should have been an instruction or a MemoryPhi");
820 return MP->getBlock();
823 // Delete a definitely dead expression, so it can be reused by the expression
824 // allocator. Some of these are not in creation functions, so we have to accept
826 void NewGVN::deleteExpression(const Expression *E) const {
827 assert(isa<BasicExpression>(E));
828 auto *BE = cast<BasicExpression>(E);
829 const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler);
830 ExpressionAllocator.Deallocate(E);
832 PHIExpression *NewGVN::createPHIExpression(Instruction *I, bool &HasBackedge,
833 bool &OriginalOpsConstant) const {
834 BasicBlock *PHIBlock = getBlockForValue(I);
835 auto *PN = cast<PHINode>(I);
837 new (ExpressionAllocator) PHIExpression(PN->getNumOperands(), PHIBlock);
839 E->allocateOperands(ArgRecycler, ExpressionAllocator);
840 E->setType(I->getType());
841 E->setOpcode(I->getOpcode());
843 // NewGVN assumes the operands of a PHI node are in a consistent order across
844 // PHIs. LLVM doesn't seem to always guarantee this. While we need to fix
845 // this in LLVM at some point we don't want GVN to find wrong congruences.
846 // Therefore, here we sort uses in predecessor order.
847 // We're sorting the values by pointer. In theory this might be cause of
848 // non-determinism, but here we don't rely on the ordering for anything
849 // significant, e.g. we don't create new instructions based on it so we're
851 SmallVector<const Use *, 4> PHIOperands;
852 for (const Use &U : PN->operands())
853 PHIOperands.push_back(&U);
854 std::sort(PHIOperands.begin(), PHIOperands.end(),
855 [&](const Use *U1, const Use *U2) {
856 return PN->getIncomingBlock(*U1) < PN->getIncomingBlock(*U2);
859 // Filter out unreachable phi operands.
860 auto Filtered = make_filter_range(PHIOperands, [&](const Use *U) {
863 if (!ReachableEdges.count({PN->getIncomingBlock(*U), PHIBlock}))
865 // Things in TOPClass are equivalent to everything.
866 if (ValueToClass.lookup(*U) == TOPClass)
870 std::transform(Filtered.begin(), Filtered.end(), op_inserter(E),
871 [&](const Use *U) -> Value * {
872 auto *BB = PN->getIncomingBlock(*U);
873 HasBackedge = HasBackedge || isBackedge(BB, PHIBlock);
874 OriginalOpsConstant =
875 OriginalOpsConstant && isa<Constant>(*U);
876 return lookupOperandLeader(*U);
881 // Set basic expression info (Arguments, type, opcode) for Expression
882 // E from Instruction I in block B.
883 bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const {
884 bool AllConstant = true;
885 if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
886 E->setType(GEP->getSourceElementType());
888 E->setType(I->getType());
889 E->setOpcode(I->getOpcode());
890 E->allocateOperands(ArgRecycler, ExpressionAllocator);
892 // Transform the operand array into an operand leader array, and keep track of
893 // whether all members are constant.
894 std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {
895 auto Operand = lookupOperandLeader(O);
896 AllConstant = AllConstant && isa<Constant>(Operand);
903 const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
906 auto *E = new (ExpressionAllocator) BasicExpression(2);
909 E->setOpcode(Opcode);
910 E->allocateOperands(ArgRecycler, ExpressionAllocator);
911 if (Instruction::isCommutative(Opcode)) {
912 // Ensure that commutative instructions that only differ by a permutation
913 // of their operands get the same value number by sorting the operand value
914 // numbers. Since all commutative instructions have two operands it is more
915 // efficient to sort by hand rather than using, say, std::sort.
916 if (shouldSwapOperands(Arg1, Arg2))
917 std::swap(Arg1, Arg2);
919 E->op_push_back(lookupOperandLeader(Arg1));
920 E->op_push_back(lookupOperandLeader(Arg2));
922 Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), SQ);
923 if (const Expression *SimplifiedE = checkSimplificationResults(E, nullptr, V))
928 // Take a Value returned by simplification of Expression E/Instruction
929 // I, and see if it resulted in a simpler expression. If so, return
931 // TODO: Once finished, this should not take an Instruction, we only
932 // use it for printing.
933 const Expression *NewGVN::checkSimplificationResults(Expression *E,
938 if (auto *C = dyn_cast<Constant>(V)) {
940 DEBUG(dbgs() << "Simplified " << *I << " to "
941 << " constant " << *C << "\n");
942 NumGVNOpsSimplified++;
943 assert(isa<BasicExpression>(E) &&
944 "We should always have had a basic expression here");
946 return createConstantExpression(C);
947 } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
949 DEBUG(dbgs() << "Simplified " << *I << " to "
950 << " variable " << *V << "\n");
952 return createVariableExpression(V);
955 CongruenceClass *CC = ValueToClass.lookup(V);
956 if (CC && CC->getDefiningExpr()) {
957 // If we simplified to something else, we need to communicate
958 // that we're users of the value we simplified to.
960 addAdditionalUsers(V, I);
962 DEBUG(dbgs() << "Simplified " << *I << " to "
963 << " expression " << *CC->getDefiningExpr() << "\n");
964 NumGVNOpsSimplified++;
966 return CC->getDefiningExpr();
971 const Expression *NewGVN::createExpression(Instruction *I) const {
972 auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
974 bool AllConstant = setBasicExpressionInfo(I, E);
976 if (I->isCommutative()) {
977 // Ensure that commutative instructions that only differ by a permutation
978 // of their operands get the same value number by sorting the operand value
979 // numbers. Since all commutative instructions have two operands it is more
980 // efficient to sort by hand rather than using, say, std::sort.
981 assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
982 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))
983 E->swapOperands(0, 1);
986 // Perform simplificaiton
987 // TODO: Right now we only check to see if we get a constant result.
988 // We may get a less than constant, but still better, result for
993 // We should handle this by simply rewriting the expression.
994 if (auto *CI = dyn_cast<CmpInst>(I)) {
995 // Sort the operand value numbers so x<y and y>x get the same value
997 CmpInst::Predicate Predicate = CI->getPredicate();
998 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) {
999 E->swapOperands(0, 1);
1000 Predicate = CmpInst::getSwappedPredicate(Predicate);
1002 E->setOpcode((CI->getOpcode() << 8) | Predicate);
1003 // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands
1004 assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
1005 "Wrong types on cmp instruction");
1006 assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
1007 E->getOperand(1)->getType() == I->getOperand(1)->getType()));
1009 SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), SQ);
1010 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1012 } else if (isa<SelectInst>(I)) {
1013 if (isa<Constant>(E->getOperand(0)) ||
1014 E->getOperand(0) == E->getOperand(1)) {
1015 assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
1016 E->getOperand(2)->getType() == I->getOperand(2)->getType());
1017 Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
1018 E->getOperand(2), SQ);
1019 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1022 } else if (I->isBinaryOp()) {
1024 SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), SQ);
1025 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1027 } else if (auto *BI = dyn_cast<BitCastInst>(I)) {
1029 SimplifyCastInst(BI->getOpcode(), BI->getOperand(0), BI->getType(), SQ);
1030 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1032 } else if (isa<GetElementPtrInst>(I)) {
1033 Value *V = SimplifyGEPInst(
1034 E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ);
1035 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1037 } else if (AllConstant) {
1038 // We don't bother trying to simplify unless all of the operands
1040 // TODO: There are a lot of Simplify*'s we could call here, if we
1041 // wanted to. The original motivating case for this code was a
1042 // zext i1 false to i8, which we don't have an interface to
1043 // simplify (IE there is no SimplifyZExt).
1045 SmallVector<Constant *, 8> C;
1046 for (Value *Arg : E->operands())
1047 C.emplace_back(cast<Constant>(Arg));
1049 if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))
1050 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1056 const AggregateValueExpression *
1057 NewGVN::createAggregateValueExpression(Instruction *I) const {
1058 if (auto *II = dyn_cast<InsertValueInst>(I)) {
1059 auto *E = new (ExpressionAllocator)
1060 AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
1061 setBasicExpressionInfo(I, E);
1062 E->allocateIntOperands(ExpressionAllocator);
1063 std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E));
1065 } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1066 auto *E = new (ExpressionAllocator)
1067 AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
1068 setBasicExpressionInfo(EI, E);
1069 E->allocateIntOperands(ExpressionAllocator);
1070 std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E));
1073 llvm_unreachable("Unhandled type of aggregate value operation");
1076 const DeadExpression *NewGVN::createDeadExpression() const {
1077 // DeadExpression has no arguments and all DeadExpression's are the same,
1078 // so we only need one of them.
1079 return SingletonDeadExpression;
1082 const VariableExpression *NewGVN::createVariableExpression(Value *V) const {
1083 auto *E = new (ExpressionAllocator) VariableExpression(V);
1084 E->setOpcode(V->getValueID());
1088 const Expression *NewGVN::createVariableOrConstant(Value *V) const {
1089 if (auto *C = dyn_cast<Constant>(V))
1090 return createConstantExpression(C);
1091 return createVariableExpression(V);
1094 const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const {
1095 auto *E = new (ExpressionAllocator) ConstantExpression(C);
1096 E->setOpcode(C->getValueID());
1100 const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const {
1101 auto *E = new (ExpressionAllocator) UnknownExpression(I);
1102 E->setOpcode(I->getOpcode());
1106 const CallExpression *
1107 NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const {
1108 // FIXME: Add operand bundles for calls.
1110 new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);
1111 setBasicExpressionInfo(CI, E);
1115 // Return true if some equivalent of instruction Inst dominates instruction U.
1116 bool NewGVN::someEquivalentDominates(const Instruction *Inst,
1117 const Instruction *U) const {
1118 auto *CC = ValueToClass.lookup(Inst);
1119 // This must be an instruction because we are only called from phi nodes
1120 // in the case that the value it needs to check against is an instruction.
1122 // The most likely candiates for dominance are the leader and the next leader.
1123 // The leader or nextleader will dominate in all cases where there is an
1124 // equivalent that is higher up in the dom tree.
1125 // We can't *only* check them, however, because the
1126 // dominator tree could have an infinite number of non-dominating siblings
1127 // with instructions that are in the right congruence class.
1132 // Instruction U could be in H, with equivalents in every other sibling.
1133 // Depending on the rpo order picked, the leader could be the equivalent in
1134 // any of these siblings.
1137 if (DT->dominates(cast<Instruction>(CC->getLeader()), U))
1139 if (CC->getNextLeader().first &&
1140 DT->dominates(cast<Instruction>(CC->getNextLeader().first), U))
1142 return llvm::any_of(*CC, [&](const Value *Member) {
1143 return Member != CC->getLeader() &&
1144 DT->dominates(cast<Instruction>(Member), U);
1148 // See if we have a congruence class and leader for this operand, and if so,
1149 // return it. Otherwise, return the operand itself.
1150 Value *NewGVN::lookupOperandLeader(Value *V) const {
1151 CongruenceClass *CC = ValueToClass.lookup(V);
1153 // Everything in TOP is represented by undef, as it can be any value.
1154 // We do have to make sure we get the type right though, so we can't set the
1155 // RepLeader to undef.
1157 return UndefValue::get(V->getType());
1158 return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
1164 const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {
1165 auto *CC = getMemoryClass(MA);
1166 assert(CC->getMemoryLeader() &&
1167 "Every MemoryAccess should be mapped to a congruence class with a "
1168 "representative memory access");
1169 return CC->getMemoryLeader();
1172 // Return true if the MemoryAccess is really equivalent to everything. This is
1173 // equivalent to the lattice value "TOP" in most lattices. This is the initial
1174 // state of all MemoryAccesses.
1175 bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const {
1176 return getMemoryClass(MA) == TOPClass;
1179 LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
1181 const MemoryAccess *MA) const {
1183 new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));
1184 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1185 E->setType(LoadType);
1187 // Give store and loads same opcode so they value number together.
1189 E->op_push_back(PointerOp);
1191 E->setAlignment(LI->getAlignment());
1193 // TODO: Value number heap versions. We may be able to discover
1194 // things alias analysis can't on it's own (IE that a store and a
1195 // load have the same value, and thus, it isn't clobbering the load).
1199 const StoreExpression *
1200 NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const {
1201 auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand());
1202 auto *E = new (ExpressionAllocator)
1203 StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);
1204 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1205 E->setType(SI->getValueOperand()->getType());
1207 // Give store and loads same opcode so they value number together.
1209 E->op_push_back(lookupOperandLeader(SI->getPointerOperand()));
1211 // TODO: Value number heap versions. We may be able to discover
1212 // things alias analysis can't on it's own (IE that a store and a
1213 // load have the same value, and thus, it isn't clobbering the load).
1217 const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {
1218 // Unlike loads, we never try to eliminate stores, so we do not check if they
1219 // are simple and avoid value numbering them.
1220 auto *SI = cast<StoreInst>(I);
1221 auto *StoreAccess = getMemoryAccess(SI);
1222 // Get the expression, if any, for the RHS of the MemoryDef.
1223 const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();
1224 if (EnableStoreRefinement)
1225 StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess);
1226 // If we bypassed the use-def chains, make sure we add a use.
1227 if (StoreRHS != StoreAccess->getDefiningAccess())
1228 addMemoryUsers(StoreRHS, StoreAccess);
1229 StoreRHS = lookupMemoryLeader(StoreRHS);
1230 // If we are defined by ourselves, use the live on entry def.
1231 if (StoreRHS == StoreAccess)
1232 StoreRHS = MSSA->getLiveOnEntryDef();
1234 if (SI->isSimple()) {
1235 // See if we are defined by a previous store expression, it already has a
1236 // value, and it's the same value as our current store. FIXME: Right now, we
1237 // only do this for simple stores, we should expand to cover memcpys, etc.
1238 const auto *LastStore = createStoreExpression(SI, StoreRHS);
1239 const auto *LastCC = ExpressionToClass.lookup(LastStore);
1240 // Basically, check if the congruence class the store is in is defined by a
1241 // store that isn't us, and has the same value. MemorySSA takes care of
1242 // ensuring the store has the same memory state as us already.
1243 // The RepStoredValue gets nulled if all the stores disappear in a class, so
1244 // we don't need to check if the class contains a store besides us.
1246 LastCC->getStoredValue() == lookupOperandLeader(SI->getValueOperand()))
1248 deleteExpression(LastStore);
1249 // Also check if our value operand is defined by a load of the same memory
1250 // location, and the memory state is the same as it was then (otherwise, it
1251 // could have been overwritten later. See test32 in
1252 // transforms/DeadStoreElimination/simple.ll).
1254 dyn_cast<LoadInst>(lookupOperandLeader(SI->getValueOperand()))) {
1255 if ((lookupOperandLeader(LI->getPointerOperand()) ==
1256 lookupOperandLeader(SI->getPointerOperand())) &&
1257 (lookupMemoryLeader(getMemoryAccess(LI)->getDefiningAccess()) ==
1259 return createStoreExpression(SI, StoreRHS);
1263 // If the store is not equivalent to anything, value number it as a store that
1264 // produces a unique memory state (instead of using it's MemoryUse, we use
1266 return createStoreExpression(SI, StoreAccess);
1269 // See if we can extract the value of a loaded pointer from a load, a store, or
1270 // a memory instruction.
1272 NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
1273 LoadInst *LI, Instruction *DepInst,
1274 MemoryAccess *DefiningAccess) const {
1275 assert((!LI || LI->isSimple()) && "Not a simple load");
1276 if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {
1277 // Can't forward from non-atomic to atomic without violating memory model.
1278 // Also don't need to coerce if they are the same type, we will just
1280 if (LI->isAtomic() > DepSI->isAtomic() ||
1281 LoadType == DepSI->getValueOperand()->getType())
1283 int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL);
1285 if (auto *C = dyn_cast<Constant>(
1286 lookupOperandLeader(DepSI->getValueOperand()))) {
1287 DEBUG(dbgs() << "Coercing load from store " << *DepSI << " to constant "
1289 return createConstantExpression(
1290 getConstantStoreValueForLoad(C, Offset, LoadType, DL));
1294 } else if (LoadInst *DepLI = dyn_cast<LoadInst>(DepInst)) {
1295 // Can't forward from non-atomic to atomic without violating memory model.
1296 if (LI->isAtomic() > DepLI->isAtomic())
1298 int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);
1300 // We can coerce a constant load into a load
1301 if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))
1302 if (auto *PossibleConstant =
1303 getConstantLoadValueForLoad(C, Offset, LoadType, DL)) {
1304 DEBUG(dbgs() << "Coercing load from load " << *LI << " to constant "
1305 << *PossibleConstant << "\n");
1306 return createConstantExpression(PossibleConstant);
1310 } else if (MemIntrinsic *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
1311 int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);
1313 if (auto *PossibleConstant =
1314 getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) {
1315 DEBUG(dbgs() << "Coercing load from meminst " << *DepMI
1316 << " to constant " << *PossibleConstant << "\n");
1317 return createConstantExpression(PossibleConstant);
1322 // All of the below are only true if the loaded pointer is produced
1323 // by the dependent instruction.
1324 if (LoadPtr != lookupOperandLeader(DepInst) &&
1325 !AA->isMustAlias(LoadPtr, DepInst))
1327 // If this load really doesn't depend on anything, then we must be loading an
1328 // undef value. This can happen when loading for a fresh allocation with no
1329 // intervening stores, for example. Note that this is only true in the case
1330 // that the result of the allocation is pointer equal to the load ptr.
1331 if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
1332 return createConstantExpression(UndefValue::get(LoadType));
1334 // If this load occurs either right after a lifetime begin,
1335 // then the loaded value is undefined.
1336 else if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {
1337 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1338 return createConstantExpression(UndefValue::get(LoadType));
1340 // If this load follows a calloc (which zero initializes memory),
1341 // then the loaded value is zero
1342 else if (isCallocLikeFn(DepInst, TLI)) {
1343 return createConstantExpression(Constant::getNullValue(LoadType));
1349 const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
1350 auto *LI = cast<LoadInst>(I);
1352 // We can eliminate in favor of non-simple loads, but we won't be able to
1353 // eliminate the loads themselves.
1354 if (!LI->isSimple())
1357 Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());
1358 // Load of undef is undef.
1359 if (isa<UndefValue>(LoadAddressLeader))
1360 return createConstantExpression(UndefValue::get(LI->getType()));
1361 MemoryAccess *OriginalAccess = getMemoryAccess(I);
1362 MemoryAccess *DefiningAccess =
1363 MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
1365 if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
1366 if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
1367 Instruction *DefiningInst = MD->getMemoryInst();
1368 // If the defining instruction is not reachable, replace with undef.
1369 if (!ReachableBlocks.count(DefiningInst->getParent()))
1370 return createConstantExpression(UndefValue::get(LI->getType()));
1371 // This will handle stores and memory insts. We only do if it the
1372 // defining access has a different type, or it is a pointer produced by
1373 // certain memory operations that cause the memory to have a fixed value
1374 // (IE things like calloc).
1375 if (const auto *CoercionResult =
1376 performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,
1377 DefiningInst, DefiningAccess))
1378 return CoercionResult;
1382 const Expression *E = createLoadExpression(LI->getType(), LoadAddressLeader,
1383 LI, DefiningAccess);
1388 NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
1389 auto *PI = PredInfo->getPredicateInfoFor(I);
1393 DEBUG(dbgs() << "Found predicate info from instruction !\n");
1395 auto *PWC = dyn_cast<PredicateWithCondition>(PI);
1399 auto *CopyOf = I->getOperand(0);
1400 auto *Cond = PWC->Condition;
1402 // If this a copy of the condition, it must be either true or false depending
1403 // on the predicate info type and edge
1404 if (CopyOf == Cond) {
1405 // We should not need to add predicate users because the predicate info is
1406 // already a use of this operand.
1407 if (isa<PredicateAssume>(PI))
1408 return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1409 if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1410 if (PBranch->TrueEdge)
1411 return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1412 return createConstantExpression(ConstantInt::getFalse(Cond->getType()));
1414 if (auto *PSwitch = dyn_cast<PredicateSwitch>(PI))
1415 return createConstantExpression(cast<Constant>(PSwitch->CaseValue));
1418 // Not a copy of the condition, so see what the predicates tell us about this
1419 // value. First, though, we check to make sure the value is actually a copy
1420 // of one of the condition operands. It's possible, in certain cases, for it
1421 // to be a copy of a predicateinfo copy. In particular, if two branch
1422 // operations use the same condition, and one branch dominates the other, we
1423 // will end up with a copy of a copy. This is currently a small deficiency in
1424 // predicateinfo. What will end up happening here is that we will value
1425 // number both copies the same anyway.
1427 // Everything below relies on the condition being a comparison.
1428 auto *Cmp = dyn_cast<CmpInst>(Cond);
1432 if (CopyOf != Cmp->getOperand(0) && CopyOf != Cmp->getOperand(1)) {
1433 DEBUG(dbgs() << "Copy is not of any condition operands!\n");
1436 Value *FirstOp = lookupOperandLeader(Cmp->getOperand(0));
1437 Value *SecondOp = lookupOperandLeader(Cmp->getOperand(1));
1438 bool SwappedOps = false;
1440 if (shouldSwapOperands(FirstOp, SecondOp)) {
1441 std::swap(FirstOp, SecondOp);
1444 CmpInst::Predicate Predicate =
1445 SwappedOps ? Cmp->getSwappedPredicate() : Cmp->getPredicate();
1447 if (isa<PredicateAssume>(PI)) {
1448 // If the comparison is true when the operands are equal, then we know the
1449 // operands are equal, because assumes must always be true.
1450 if (CmpInst::isTrueWhenEqual(Predicate)) {
1451 addPredicateUsers(PI, I);
1452 addAdditionalUsers(Cmp->getOperand(0), I);
1453 return createVariableOrConstant(FirstOp);
1456 if (const auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1457 // If we are *not* a copy of the comparison, we may equal to the other
1458 // operand when the predicate implies something about equality of
1459 // operations. In particular, if the comparison is true/false when the
1460 // operands are equal, and we are on the right edge, we know this operation
1461 // is equal to something.
1462 if ((PBranch->TrueEdge && Predicate == CmpInst::ICMP_EQ) ||
1463 (!PBranch->TrueEdge && Predicate == CmpInst::ICMP_NE)) {
1464 addPredicateUsers(PI, I);
1465 addAdditionalUsers(Cmp->getOperand(0), I);
1466 return createVariableOrConstant(FirstOp);
1468 // Handle the special case of floating point.
1469 if (((PBranch->TrueEdge && Predicate == CmpInst::FCMP_OEQ) ||
1470 (!PBranch->TrueEdge && Predicate == CmpInst::FCMP_UNE)) &&
1471 isa<ConstantFP>(FirstOp) && !cast<ConstantFP>(FirstOp)->isZero()) {
1472 addPredicateUsers(PI, I);
1473 addAdditionalUsers(Cmp->getOperand(0), I);
1474 return createConstantExpression(cast<Constant>(FirstOp));
1480 // Evaluate read only and pure calls, and create an expression result.
1481 const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
1482 auto *CI = cast<CallInst>(I);
1483 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1484 // Instrinsics with the returned attribute are copies of arguments.
1485 if (auto *ReturnedValue = II->getReturnedArgOperand()) {
1486 if (II->getIntrinsicID() == Intrinsic::ssa_copy)
1487 if (const auto *Result = performSymbolicPredicateInfoEvaluation(I))
1489 return createVariableOrConstant(ReturnedValue);
1492 if (AA->doesNotAccessMemory(CI)) {
1493 return createCallExpression(CI, TOPClass->getMemoryLeader());
1494 } else if (AA->onlyReadsMemory(CI)) {
1495 MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI);
1496 return createCallExpression(CI, DefiningAccess);
1501 // Retrieve the memory class for a given MemoryAccess.
1502 CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {
1504 auto *Result = MemoryAccessToClass.lookup(MA);
1505 assert(Result && "Should have found memory class");
1509 // Update the MemoryAccess equivalence table to say that From is equal to To,
1510 // and return true if this is different from what already existed in the table.
1511 bool NewGVN::setMemoryClass(const MemoryAccess *From,
1512 CongruenceClass *NewClass) {
1514 "Every MemoryAccess should be getting mapped to a non-null class");
1515 DEBUG(dbgs() << "Setting " << *From);
1516 DEBUG(dbgs() << " equivalent to congruence class ");
1517 DEBUG(dbgs() << NewClass->getID() << " with current MemoryAccess leader ");
1518 DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n");
1520 auto LookupResult = MemoryAccessToClass.find(From);
1521 bool Changed = false;
1522 // If it's already in the table, see if the value changed.
1523 if (LookupResult != MemoryAccessToClass.end()) {
1524 auto *OldClass = LookupResult->second;
1525 if (OldClass != NewClass) {
1526 // If this is a phi, we have to handle memory member updates.
1527 if (auto *MP = dyn_cast<MemoryPhi>(From)) {
1528 OldClass->memory_erase(MP);
1529 NewClass->memory_insert(MP);
1530 // This may have killed the class if it had no non-memory members
1531 if (OldClass->getMemoryLeader() == From) {
1532 if (OldClass->definesNoMemory()) {
1533 OldClass->setMemoryLeader(nullptr);
1535 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1536 DEBUG(dbgs() << "Memory class leader change for class "
1537 << OldClass->getID() << " to "
1538 << *OldClass->getMemoryLeader()
1539 << " due to removal of a memory member " << *From
1541 markMemoryLeaderChangeTouched(OldClass);
1545 // It wasn't equivalent before, and now it is.
1546 LookupResult->second = NewClass;
1554 // Determine if a instruction is cycle-free. That means the values in the
1555 // instruction don't depend on any expressions that can change value as a result
1556 // of the instruction. For example, a non-cycle free instruction would be v =
1558 bool NewGVN::isCycleFree(const Instruction *I) const {
1559 // In order to compute cycle-freeness, we do SCC finding on the instruction,
1560 // and see what kind of SCC it ends up in. If it is a singleton, it is
1561 // cycle-free. If it is not in a singleton, it is only cycle free if the
1562 // other members are all phi nodes (as they do not compute anything, they are
1564 auto ICS = InstCycleState.lookup(I);
1565 if (ICS == ICS_Unknown) {
1567 auto &SCC = SCCFinder.getComponentFor(I);
1568 // It's cycle free if it's size 1 or or the SCC is *only* phi nodes.
1569 if (SCC.size() == 1)
1570 InstCycleState.insert({I, ICS_CycleFree});
1573 llvm::all_of(SCC, [](const Value *V) { return isa<PHINode>(V); });
1574 ICS = AllPhis ? ICS_CycleFree : ICS_Cycle;
1575 for (auto *Member : SCC)
1576 if (auto *MemberPhi = dyn_cast<PHINode>(Member))
1577 InstCycleState.insert({MemberPhi, ICS});
1580 if (ICS == ICS_Cycle)
1585 // Evaluate PHI nodes symbolically, and create an expression result.
1586 const Expression *NewGVN::performSymbolicPHIEvaluation(Instruction *I) const {
1587 // Resolve irreducible and reducible phi cycles.
1588 // FIXME: This is hopefully a temporary solution while we resolve the issues
1589 // with fixpointing self-cycles. It currently should be "guaranteed" to be
1590 // correct, but non-optimal. The SCCFinder does not, for example, take
1591 // reachability of arguments into account, etc.
1593 bool CanOptimize = true;
1594 SmallPtrSet<Value *, 8> OuterOps;
1596 auto &Component = SCCFinder.getComponentFor(I);
1597 for (auto *Member : Component) {
1598 if (!isa<PHINode>(Member)) {
1599 CanOptimize = false;
1602 for (auto &PHIOp : cast<PHINode>(Member)->operands())
1603 if (!isa<PHINode>(PHIOp) || !Component.count(cast<PHINode>(PHIOp)))
1604 OuterOps.insert(PHIOp);
1606 if (CanOptimize && OuterOps.size() == 1) {
1607 DEBUG(dbgs() << "Resolving cyclic phi to value " << *(*OuterOps.begin())
1609 return createVariableOrConstant(*OuterOps.begin());
1611 // True if one of the incoming phi edges is a backedge.
1612 bool HasBackedge = false;
1613 // All constant tracks the state of whether all the *original* phi operands
1614 // This is really shorthand for "this phi cannot cycle due to forward
1615 // change in value of the phi is guaranteed not to later change the value of
1616 // the phi. IE it can't be v = phi(undef, v+1)
1617 bool AllConstant = true;
1619 cast<PHIExpression>(createPHIExpression(I, HasBackedge, AllConstant));
1620 // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
1621 // See if all arguments are the same.
1622 // We track if any were undef because they need special handling.
1623 bool HasUndef = false;
1624 auto Filtered = make_filter_range(E->operands(), [&](Value *Arg) {
1625 if (isa<UndefValue>(Arg)) {
1631 // If we are left with no operands, it's dead.
1632 if (Filtered.begin() == Filtered.end()) {
1633 // If it has undef at this point, it means there are no-non-undef arguments,
1634 // and thus, the value of the phi node must be undef.
1636 DEBUG(dbgs() << "PHI Node " << *I
1637 << " has no non-undef arguments, valuing it as undef\n");
1638 return createConstantExpression(UndefValue::get(I->getType()));
1641 DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n");
1642 deleteExpression(E);
1643 return createDeadExpression();
1645 unsigned NumOps = 0;
1646 Value *AllSameValue = *(Filtered.begin());
1648 // Can't use std::equal here, sadly, because filter.begin moves.
1649 if (llvm::all_of(Filtered, [&](Value *Arg) {
1651 return Arg == AllSameValue;
1653 // In LLVM's non-standard representation of phi nodes, it's possible to have
1654 // phi nodes with cycles (IE dependent on other phis that are .... dependent
1655 // on the original phi node), especially in weird CFG's where some arguments
1656 // are unreachable, or uninitialized along certain paths. This can cause
1657 // infinite loops during evaluation. We work around this by not trying to
1658 // really evaluate them independently, but instead using a variable
1659 // expression to say if one is equivalent to the other.
1660 // We also special case undef, so that if we have an undef, we can't use the
1661 // common value unless it dominates the phi block.
1663 // If we have undef and at least one other value, this is really a
1664 // multivalued phi, and we need to know if it's cycle free in order to
1665 // evaluate whether we can ignore the undef. The other parts of this are
1666 // just shortcuts. If there is no backedge, or all operands are
1667 // constants, or all operands are ignored but the undef, it also must be
1669 if (!AllConstant && HasBackedge && NumOps > 0 &&
1670 !isa<UndefValue>(AllSameValue) && !isCycleFree(I))
1673 // Only have to check for instructions
1674 if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))
1675 if (!someEquivalentDominates(AllSameInst, I))
1679 NumGVNPhisAllSame++;
1680 DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
1682 deleteExpression(E);
1683 return createVariableOrConstant(AllSameValue);
1689 NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {
1690 if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1691 auto *II = dyn_cast<IntrinsicInst>(EI->getAggregateOperand());
1692 if (II && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) {
1693 unsigned Opcode = 0;
1694 // EI might be an extract from one of our recognised intrinsics. If it
1695 // is we'll synthesize a semantically equivalent expression instead on
1696 // an extract value expression.
1697 switch (II->getIntrinsicID()) {
1698 case Intrinsic::sadd_with_overflow:
1699 case Intrinsic::uadd_with_overflow:
1700 Opcode = Instruction::Add;
1702 case Intrinsic::ssub_with_overflow:
1703 case Intrinsic::usub_with_overflow:
1704 Opcode = Instruction::Sub;
1706 case Intrinsic::smul_with_overflow:
1707 case Intrinsic::umul_with_overflow:
1708 Opcode = Instruction::Mul;
1715 // Intrinsic recognized. Grab its args to finish building the
1717 assert(II->getNumArgOperands() == 2 &&
1718 "Expect two args for recognised intrinsics.");
1719 return createBinaryExpression(
1720 Opcode, EI->getType(), II->getArgOperand(0), II->getArgOperand(1));
1725 return createAggregateValueExpression(I);
1727 const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
1728 auto *CI = dyn_cast<CmpInst>(I);
1729 // See if our operands are equal to those of a previous predicate, and if so,
1730 // if it implies true or false.
1731 auto Op0 = lookupOperandLeader(CI->getOperand(0));
1732 auto Op1 = lookupOperandLeader(CI->getOperand(1));
1733 auto OurPredicate = CI->getPredicate();
1734 if (shouldSwapOperands(Op0, Op1)) {
1735 std::swap(Op0, Op1);
1736 OurPredicate = CI->getSwappedPredicate();
1739 // Avoid processing the same info twice
1740 const PredicateBase *LastPredInfo = nullptr;
1741 // See if we know something about the comparison itself, like it is the target
1743 auto *CmpPI = PredInfo->getPredicateInfoFor(I);
1744 if (dyn_cast_or_null<PredicateAssume>(CmpPI))
1745 return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1748 // This condition does not depend on predicates, no need to add users
1749 if (CI->isTrueWhenEqual())
1750 return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1751 else if (CI->isFalseWhenEqual())
1752 return createConstantExpression(ConstantInt::getFalse(CI->getType()));
1755 // NOTE: Because we are comparing both operands here and below, and using
1756 // previous comparisons, we rely on fact that predicateinfo knows to mark
1757 // comparisons that use renamed operands as users of the earlier comparisons.
1758 // It is *not* enough to just mark predicateinfo renamed operands as users of
1759 // the earlier comparisons, because the *other* operand may have changed in a
1760 // previous iteration.
1763 // %b.0 = ssa.copy(%b)
1765 // icmp slt %c, %b.0
1767 // %c and %a may start out equal, and thus, the code below will say the second
1768 // %icmp is false. c may become equal to something else, and in that case the
1769 // %second icmp *must* be reexamined, but would not if only the renamed
1770 // %operands are considered users of the icmp.
1772 // *Currently* we only check one level of comparisons back, and only mark one
1773 // level back as touched when changes appen . If you modify this code to look
1774 // back farther through comparisons, you *must* mark the appropriate
1775 // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if
1776 // we know something just from the operands themselves
1778 // See if our operands have predicate info, so that we may be able to derive
1779 // something from a previous comparison.
1780 for (const auto &Op : CI->operands()) {
1781 auto *PI = PredInfo->getPredicateInfoFor(Op);
1782 if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) {
1783 if (PI == LastPredInfo)
1787 // TODO: Along the false edge, we may know more things too, like icmp of
1788 // same operands is false.
1789 // TODO: We only handle actual comparison conditions below, not and/or.
1790 auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition);
1793 auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0));
1794 auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1));
1795 auto BranchPredicate = BranchCond->getPredicate();
1796 if (shouldSwapOperands(BranchOp0, BranchOp1)) {
1797 std::swap(BranchOp0, BranchOp1);
1798 BranchPredicate = BranchCond->getSwappedPredicate();
1800 if (BranchOp0 == Op0 && BranchOp1 == Op1) {
1801 if (PBranch->TrueEdge) {
1802 // If we know the previous predicate is true and we are in the true
1803 // edge then we may be implied true or false.
1804 if (CmpInst::isImpliedTrueByMatchingCmp(BranchPredicate,
1806 addPredicateUsers(PI, I);
1807 return createConstantExpression(
1808 ConstantInt::getTrue(CI->getType()));
1811 if (CmpInst::isImpliedFalseByMatchingCmp(BranchPredicate,
1813 addPredicateUsers(PI, I);
1814 return createConstantExpression(
1815 ConstantInt::getFalse(CI->getType()));
1819 // Just handle the ne and eq cases, where if we have the same
1820 // operands, we may know something.
1821 if (BranchPredicate == OurPredicate) {
1822 addPredicateUsers(PI, I);
1823 // Same predicate, same ops,we know it was false, so this is false.
1824 return createConstantExpression(
1825 ConstantInt::getFalse(CI->getType()));
1826 } else if (BranchPredicate ==
1827 CmpInst::getInversePredicate(OurPredicate)) {
1828 addPredicateUsers(PI, I);
1829 // Inverse predicate, we know the other was false, so this is true.
1830 return createConstantExpression(
1831 ConstantInt::getTrue(CI->getType()));
1837 // Create expression will take care of simplifyCmpInst
1838 return createExpression(I);
1841 // Return true if V is a value that will always be available (IE can
1842 // be placed anywhere) in the function. We don't do globals here
1843 // because they are often worse to put in place.
1844 // TODO: Separate cost from availability
1845 static bool alwaysAvailable(Value *V) {
1846 return isa<Constant>(V) || isa<Argument>(V);
1849 // Substitute and symbolize the value before value numbering.
1851 NewGVN::performSymbolicEvaluation(Value *V,
1852 SmallPtrSetImpl<Value *> &Visited) const {
1853 const Expression *E = nullptr;
1854 if (auto *C = dyn_cast<Constant>(V))
1855 E = createConstantExpression(C);
1856 else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
1857 E = createVariableExpression(V);
1859 // TODO: memory intrinsics.
1860 // TODO: Some day, we should do the forward propagation and reassociation
1861 // parts of the algorithm.
1862 auto *I = cast<Instruction>(V);
1863 switch (I->getOpcode()) {
1864 case Instruction::ExtractValue:
1865 case Instruction::InsertValue:
1866 E = performSymbolicAggrValueEvaluation(I);
1868 case Instruction::PHI:
1869 E = performSymbolicPHIEvaluation(I);
1871 case Instruction::Call:
1872 E = performSymbolicCallEvaluation(I);
1874 case Instruction::Store:
1875 E = performSymbolicStoreEvaluation(I);
1877 case Instruction::Load:
1878 E = performSymbolicLoadEvaluation(I);
1880 case Instruction::BitCast: {
1881 E = createExpression(I);
1883 case Instruction::ICmp:
1884 case Instruction::FCmp: {
1885 E = performSymbolicCmpEvaluation(I);
1887 case Instruction::Add:
1888 case Instruction::FAdd:
1889 case Instruction::Sub:
1890 case Instruction::FSub:
1891 case Instruction::Mul:
1892 case Instruction::FMul:
1893 case Instruction::UDiv:
1894 case Instruction::SDiv:
1895 case Instruction::FDiv:
1896 case Instruction::URem:
1897 case Instruction::SRem:
1898 case Instruction::FRem:
1899 case Instruction::Shl:
1900 case Instruction::LShr:
1901 case Instruction::AShr:
1902 case Instruction::And:
1903 case Instruction::Or:
1904 case Instruction::Xor:
1905 case Instruction::Trunc:
1906 case Instruction::ZExt:
1907 case Instruction::SExt:
1908 case Instruction::FPToUI:
1909 case Instruction::FPToSI:
1910 case Instruction::UIToFP:
1911 case Instruction::SIToFP:
1912 case Instruction::FPTrunc:
1913 case Instruction::FPExt:
1914 case Instruction::PtrToInt:
1915 case Instruction::IntToPtr:
1916 case Instruction::Select:
1917 case Instruction::ExtractElement:
1918 case Instruction::InsertElement:
1919 case Instruction::ShuffleVector:
1920 case Instruction::GetElementPtr:
1921 E = createExpression(I);
1930 // Look up a container in a map, and then call a function for each thing in the
1932 template <typename Map, typename KeyType, typename Func>
1933 void NewGVN::for_each_found(Map &M, const KeyType &Key, Func F) {
1934 const auto Result = M.find_as(Key);
1935 if (Result != M.end())
1936 for (typename Map::mapped_type::value_type Mapped : Result->second)
1940 // Look up a container of values/instructions in a map, and touch all the
1941 // instructions in the container. Then erase value from the map.
1942 template <typename Map, typename KeyType>
1943 void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
1944 const auto Result = M.find_as(Key);
1945 if (Result != M.end()) {
1946 for (const typename Map::mapped_type::value_type Mapped : Result->second)
1947 TouchedInstructions.set(InstrToDFSNum(Mapped));
1952 void NewGVN::addAdditionalUsers(Value *To, Value *User) const {
1953 AdditionalUsers[To].insert(User);
1956 void NewGVN::markUsersTouched(Value *V) {
1957 // Now mark the users as touched.
1958 for (auto *User : V->users()) {
1959 assert(isa<Instruction>(User) && "Use of value not within an instruction?");
1960 TouchedInstructions.set(InstrToDFSNum(User));
1962 touchAndErase(AdditionalUsers, V);
1965 void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const {
1966 DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");
1967 MemoryToUsers[To].insert(U);
1970 void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {
1971 TouchedInstructions.set(MemoryToDFSNum(MA));
1974 void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
1975 if (isa<MemoryUse>(MA))
1977 for (auto U : MA->users())
1978 TouchedInstructions.set(MemoryToDFSNum(U));
1979 touchAndErase(MemoryToUsers, MA);
1982 // Add I to the set of users of a given predicate.
1983 void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) const {
1984 // Don't add temporary instructions to the user lists.
1985 if (AllTempInstructions.count(I))
1988 if (auto *PBranch = dyn_cast<PredicateBranch>(PB))
1989 PredicateToUsers[PBranch->Condition].insert(I);
1990 else if (auto *PAssume = dyn_cast<PredicateBranch>(PB))
1991 PredicateToUsers[PAssume->Condition].insert(I);
1994 // Touch all the predicates that depend on this instruction.
1995 void NewGVN::markPredicateUsersTouched(Instruction *I) {
1996 touchAndErase(PredicateToUsers, I);
1999 // Mark users affected by a memory leader change.
2000 void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {
2001 for (auto M : CC->memory())
2002 markMemoryDefTouched(M);
2005 // Touch the instructions that need to be updated after a congruence class has a
2006 // leader change, and mark changed values.
2007 void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {
2008 for (auto M : *CC) {
2009 if (auto *I = dyn_cast<Instruction>(M))
2010 TouchedInstructions.set(InstrToDFSNum(I));
2011 LeaderChanges.insert(M);
2015 // Give a range of things that have instruction DFS numbers, this will return
2016 // the member of the range with the smallest dfs number.
2017 template <class T, class Range>
2018 T *NewGVN::getMinDFSOfRange(const Range &R) const {
2019 std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2020 for (const auto X : R) {
2021 auto DFSNum = InstrToDFSNum(X);
2022 if (DFSNum < MinDFS.second)
2023 MinDFS = {X, DFSNum};
2025 return MinDFS.first;
2028 // This function returns the MemoryAccess that should be the next leader of
2029 // congruence class CC, under the assumption that the current leader is going to
2031 const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {
2032 // TODO: If this ends up to slow, we can maintain a next memory leader like we
2033 // do for regular leaders.
2034 // Make sure there will be a leader to find
2035 assert(!CC->definesNoMemory() && "Can't get next leader if there is none");
2036 if (CC->getStoreCount() > 0) {
2037 if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first))
2038 return getMemoryAccess(NL);
2039 // Find the store with the minimum DFS number.
2040 auto *V = getMinDFSOfRange<Value>(make_filter_range(
2041 *CC, [&](const Value *V) { return isa<StoreInst>(V); }));
2042 return getMemoryAccess(cast<StoreInst>(V));
2044 assert(CC->getStoreCount() == 0);
2046 // Given our assertion, hitting this part must mean
2047 // !OldClass->memory_empty()
2048 if (CC->memory_size() == 1)
2049 return *CC->memory_begin();
2050 return getMinDFSOfRange<const MemoryPhi>(CC->memory());
2053 // This function returns the next value leader of a congruence class, under the
2054 // assumption that the current leader is going away. This should end up being
2055 // the next most dominating member.
2056 Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {
2057 // We don't need to sort members if there is only 1, and we don't care about
2058 // sorting the TOP class because everything either gets out of it or is
2061 if (CC->size() == 1 || CC == TOPClass) {
2062 return *(CC->begin());
2063 } else if (CC->getNextLeader().first) {
2064 ++NumGVNAvoidedSortedLeaderChanges;
2065 return CC->getNextLeader().first;
2067 ++NumGVNSortedLeaderChanges;
2068 // NOTE: If this ends up to slow, we can maintain a dual structure for
2069 // member testing/insertion, or keep things mostly sorted, and sort only
2070 // here, or use SparseBitVector or ....
2071 return getMinDFSOfRange<Value>(*CC);
2075 // Move a MemoryAccess, currently in OldClass, to NewClass, including updates to
2076 // the memory members, etc for the move.
2078 // The invariants of this function are:
2080 // I must be moving to NewClass from OldClass The StoreCount of OldClass and
2081 // NewClass is expected to have been updated for I already if it is is a store.
2082 // The OldClass memory leader has not been updated yet if I was the leader.
2083 void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,
2084 MemoryAccess *InstMA,
2085 CongruenceClass *OldClass,
2086 CongruenceClass *NewClass) {
2087 // If the leader is I, and we had a represenative MemoryAccess, it should
2088 // be the MemoryAccess of OldClass.
2089 assert((!InstMA || !OldClass->getMemoryLeader() ||
2090 OldClass->getLeader() != I ||
2091 OldClass->getMemoryLeader() == InstMA) &&
2092 "Representative MemoryAccess mismatch");
2093 // First, see what happens to the new class
2094 if (!NewClass->getMemoryLeader()) {
2095 // Should be a new class, or a store becoming a leader of a new class.
2096 assert(NewClass->size() == 1 ||
2097 (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));
2098 NewClass->setMemoryLeader(InstMA);
2099 // Mark it touched if we didn't just create a singleton
2100 DEBUG(dbgs() << "Memory class leader change for class " << NewClass->getID()
2101 << " due to new memory instruction becoming leader\n");
2102 markMemoryLeaderChangeTouched(NewClass);
2104 setMemoryClass(InstMA, NewClass);
2105 // Now, fixup the old class if necessary
2106 if (OldClass->getMemoryLeader() == InstMA) {
2107 if (!OldClass->definesNoMemory()) {
2108 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
2109 DEBUG(dbgs() << "Memory class leader change for class "
2110 << OldClass->getID() << " to "
2111 << *OldClass->getMemoryLeader()
2112 << " due to removal of old leader " << *InstMA << "\n");
2113 markMemoryLeaderChangeTouched(OldClass);
2115 OldClass->setMemoryLeader(nullptr);
2119 // Move a value, currently in OldClass, to be part of NewClass
2120 // Update OldClass and NewClass for the move (including changing leaders, etc).
2121 void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,
2122 CongruenceClass *OldClass,
2123 CongruenceClass *NewClass) {
2124 if (I == OldClass->getNextLeader().first)
2125 OldClass->resetNextLeader();
2128 NewClass->insert(I);
2130 if (NewClass->getLeader() != I)
2131 NewClass->addPossibleNextLeader({I, InstrToDFSNum(I)});
2132 // Handle our special casing of stores.
2133 if (auto *SI = dyn_cast<StoreInst>(I)) {
2134 OldClass->decStoreCount();
2135 // Okay, so when do we want to make a store a leader of a class?
2136 // If we have a store defined by an earlier load, we want the earlier load
2137 // to lead the class.
2138 // If we have a store defined by something else, we want the store to lead
2139 // the class so everything else gets the "something else" as a value.
2140 // If we have a store as the single member of the class, we want the store
2142 if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) {
2143 // If it's a store expression we are using, it means we are not equivalent
2144 // to something earlier.
2145 if (auto *SE = dyn_cast<StoreExpression>(E)) {
2146 NewClass->setStoredValue(SE->getStoredValue());
2147 markValueLeaderChangeTouched(NewClass);
2148 // Shift the new class leader to be the store
2149 DEBUG(dbgs() << "Changing leader of congruence class "
2150 << NewClass->getID() << " from " << *NewClass->getLeader()
2151 << " to " << *SI << " because store joined class\n");
2152 // If we changed the leader, we have to mark it changed because we don't
2153 // know what it will do to symbolic evlauation.
2154 NewClass->setLeader(SI);
2156 // We rely on the code below handling the MemoryAccess change.
2158 NewClass->incStoreCount();
2160 // True if there is no memory instructions left in a class that had memory
2161 // instructions before.
2163 // If it's not a memory use, set the MemoryAccess equivalence
2164 auto *InstMA = dyn_cast_or_null<MemoryDef>(getMemoryAccess(I));
2166 moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);
2167 ValueToClass[I] = NewClass;
2168 // See if we destroyed the class or need to swap leaders.
2169 if (OldClass->empty() && OldClass != TOPClass) {
2170 if (OldClass->getDefiningExpr()) {
2171 DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()
2172 << " from table\n");
2173 ExpressionToClass.erase(OldClass->getDefiningExpr());
2175 } else if (OldClass->getLeader() == I) {
2176 // When the leader changes, the value numbering of
2177 // everything may change due to symbolization changes, so we need to
2179 DEBUG(dbgs() << "Value class leader change for class " << OldClass->getID()
2181 ++NumGVNLeaderChanges;
2182 // Destroy the stored value if there are no more stores to represent it.
2183 // Note that this is basically clean up for the expression removal that
2184 // happens below. If we remove stores from a class, we may leave it as a
2185 // class of equivalent memory phis.
2186 if (OldClass->getStoreCount() == 0) {
2187 if (OldClass->getStoredValue())
2188 OldClass->setStoredValue(nullptr);
2190 OldClass->setLeader(getNextValueLeader(OldClass));
2191 OldClass->resetNextLeader();
2192 markValueLeaderChangeTouched(OldClass);
2196 // For a given expression, mark the phi of ops instructions that could have
2197 // changed as a result.
2198 void NewGVN::markPhiOfOpsChanged(const HashedExpression &HE) {
2199 touchAndErase(ExpressionToPhiOfOps, HE);
2202 // Perform congruence finding on a given value numbering expression.
2203 void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
2204 // This is guaranteed to return something, since it will at least find
2207 CongruenceClass *IClass = ValueToClass.lookup(I);
2208 assert(IClass && "Should have found a IClass");
2209 // Dead classes should have been eliminated from the mapping.
2210 assert(!IClass->isDead() && "Found a dead class");
2212 CongruenceClass *EClass = nullptr;
2213 HashedExpression HE(E);
2214 if (const auto *VE = dyn_cast<VariableExpression>(E)) {
2215 EClass = ValueToClass.lookup(VE->getVariableValue());
2216 } else if (isa<DeadExpression>(E)) {
2220 auto lookupResult = ExpressionToClass.insert_as({E, nullptr}, HE);
2222 // If it's not in the value table, create a new congruence class.
2223 if (lookupResult.second) {
2224 CongruenceClass *NewClass = createCongruenceClass(nullptr, E);
2225 auto place = lookupResult.first;
2226 place->second = NewClass;
2228 // Constants and variables should always be made the leader.
2229 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2230 NewClass->setLeader(CE->getConstantValue());
2231 } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {
2232 StoreInst *SI = SE->getStoreInst();
2233 NewClass->setLeader(SI);
2234 NewClass->setStoredValue(SE->getStoredValue());
2235 // The RepMemoryAccess field will be filled in properly by the
2236 // moveValueToNewCongruenceClass call.
2238 NewClass->setLeader(I);
2240 assert(!isa<VariableExpression>(E) &&
2241 "VariableExpression should have been handled already");
2244 DEBUG(dbgs() << "Created new congruence class for " << *I
2245 << " using expression " << *E << " at " << NewClass->getID()
2246 << " and leader " << *(NewClass->getLeader()));
2247 if (NewClass->getStoredValue())
2248 DEBUG(dbgs() << " and stored value " << *(NewClass->getStoredValue()));
2249 DEBUG(dbgs() << "\n");
2251 EClass = lookupResult.first->second;
2252 if (isa<ConstantExpression>(E))
2253 assert((isa<Constant>(EClass->getLeader()) ||
2254 (EClass->getStoredValue() &&
2255 isa<Constant>(EClass->getStoredValue()))) &&
2256 "Any class with a constant expression should have a "
2259 assert(EClass && "Somehow don't have an eclass");
2261 assert(!EClass->isDead() && "We accidentally looked up a dead class");
2264 bool ClassChanged = IClass != EClass;
2265 bool LeaderChanged = LeaderChanges.erase(I);
2266 if (ClassChanged || LeaderChanged) {
2267 DEBUG(dbgs() << "New class " << EClass->getID() << " for expression " << *E
2270 moveValueToNewCongruenceClass(I, E, IClass, EClass);
2271 markPhiOfOpsChanged(HE);
2274 markUsersTouched(I);
2275 if (MemoryAccess *MA = getMemoryAccess(I))
2276 markMemoryUsersTouched(MA);
2277 if (auto *CI = dyn_cast<CmpInst>(I))
2278 markPredicateUsersTouched(CI);
2280 // If we changed the class of the store, we want to ensure nothing finds the
2281 // old store expression. In particular, loads do not compare against stored
2282 // value, so they will find old store expressions (and associated class
2283 // mappings) if we leave them in the table.
2284 if (ClassChanged && isa<StoreInst>(I)) {
2285 auto *OldE = ValueToExpression.lookup(I);
2286 // It could just be that the old class died. We don't want to erase it if we
2287 // just moved classes.
2288 if (OldE && isa<StoreExpression>(OldE) && *E != *OldE)
2289 ExpressionToClass.erase(OldE);
2291 ValueToExpression[I] = E;
2294 // Process the fact that Edge (from, to) is reachable, including marking
2295 // any newly reachable blocks and instructions for processing.
2296 void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
2297 // Check if the Edge was reachable before.
2298 if (ReachableEdges.insert({From, To}).second) {
2299 // If this block wasn't reachable before, all instructions are touched.
2300 if (ReachableBlocks.insert(To).second) {
2301 DEBUG(dbgs() << "Block " << getBlockName(To) << " marked reachable\n");
2302 const auto &InstRange = BlockInstRange.lookup(To);
2303 TouchedInstructions.set(InstRange.first, InstRange.second);
2305 DEBUG(dbgs() << "Block " << getBlockName(To)
2306 << " was reachable, but new edge {" << getBlockName(From)
2307 << "," << getBlockName(To) << "} to it found\n");
2309 // We've made an edge reachable to an existing block, which may
2310 // impact predicates. Otherwise, only mark the phi nodes as touched, as
2311 // they are the only thing that depend on new edges. Anything using their
2312 // values will get propagated to if necessary.
2313 if (MemoryAccess *MemPhi = getMemoryAccess(To))
2314 TouchedInstructions.set(InstrToDFSNum(MemPhi));
2316 auto BI = To->begin();
2317 while (isa<PHINode>(BI)) {
2318 TouchedInstructions.set(InstrToDFSNum(&*BI));
2321 for_each_found(PHIOfOpsPHIs, To, [&](const PHINode *I) {
2322 TouchedInstructions.set(InstrToDFSNum(I));
2328 // Given a predicate condition (from a switch, cmp, or whatever) and a block,
2329 // see if we know some constant value for it already.
2330 Value *NewGVN::findConditionEquivalence(Value *Cond) const {
2331 auto Result = lookupOperandLeader(Cond);
2332 if (isa<Constant>(Result))
2337 // Process the outgoing edges of a block for reachability.
2338 void NewGVN::processOutgoingEdges(TerminatorInst *TI, BasicBlock *B) {
2339 // Evaluate reachability of terminator instruction.
2341 if ((BR = dyn_cast<BranchInst>(TI)) && BR->isConditional()) {
2342 Value *Cond = BR->getCondition();
2343 Value *CondEvaluated = findConditionEquivalence(Cond);
2344 if (!CondEvaluated) {
2345 if (auto *I = dyn_cast<Instruction>(Cond)) {
2346 const Expression *E = createExpression(I);
2347 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2348 CondEvaluated = CE->getConstantValue();
2350 } else if (isa<ConstantInt>(Cond)) {
2351 CondEvaluated = Cond;
2355 BasicBlock *TrueSucc = BR->getSuccessor(0);
2356 BasicBlock *FalseSucc = BR->getSuccessor(1);
2357 if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) {
2359 DEBUG(dbgs() << "Condition for Terminator " << *TI
2360 << " evaluated to true\n");
2361 updateReachableEdge(B, TrueSucc);
2362 } else if (CI->isZero()) {
2363 DEBUG(dbgs() << "Condition for Terminator " << *TI
2364 << " evaluated to false\n");
2365 updateReachableEdge(B, FalseSucc);
2368 updateReachableEdge(B, TrueSucc);
2369 updateReachableEdge(B, FalseSucc);
2371 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
2372 // For switches, propagate the case values into the case
2375 // Remember how many outgoing edges there are to every successor.
2376 SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges;
2378 Value *SwitchCond = SI->getCondition();
2379 Value *CondEvaluated = findConditionEquivalence(SwitchCond);
2380 // See if we were able to turn this switch statement into a constant.
2381 if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) {
2382 auto *CondVal = cast<ConstantInt>(CondEvaluated);
2383 // We should be able to get case value for this.
2384 auto Case = *SI->findCaseValue(CondVal);
2385 if (Case.getCaseSuccessor() == SI->getDefaultDest()) {
2386 // We proved the value is outside of the range of the case.
2387 // We can't do anything other than mark the default dest as reachable,
2389 updateReachableEdge(B, SI->getDefaultDest());
2392 // Now get where it goes and mark it reachable.
2393 BasicBlock *TargetBlock = Case.getCaseSuccessor();
2394 updateReachableEdge(B, TargetBlock);
2396 for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; ++i) {
2397 BasicBlock *TargetBlock = SI->getSuccessor(i);
2398 ++SwitchEdges[TargetBlock];
2399 updateReachableEdge(B, TargetBlock);
2403 // Otherwise this is either unconditional, or a type we have no
2404 // idea about. Just mark successors as reachable.
2405 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
2406 BasicBlock *TargetBlock = TI->getSuccessor(i);
2407 updateReachableEdge(B, TargetBlock);
2410 // This also may be a memory defining terminator, in which case, set it
2411 // equivalent only to itself.
2413 auto *MA = getMemoryAccess(TI);
2414 if (MA && !isa<MemoryUse>(MA)) {
2415 auto *CC = ensureLeaderOfMemoryClass(MA);
2416 if (setMemoryClass(MA, CC))
2417 markMemoryUsersTouched(MA);
2422 void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB,
2423 Instruction *ExistingValue) {
2424 InstrDFS[Op] = InstrToDFSNum(ExistingValue);
2425 AllTempInstructions.insert(Op);
2426 PHIOfOpsPHIs[BB].push_back(Op);
2427 TempToBlock[Op] = BB;
2429 RealToTemp[ExistingValue] = Op;
2432 static bool okayForPHIOfOps(const Instruction *I) {
2433 return isa<BinaryOperator>(I) || isa<SelectInst>(I) || isa<CmpInst>(I) ||
2437 // When we see an instruction that is an op of phis, generate the equivalent phi
2440 NewGVN::makePossiblePhiOfOps(Instruction *I, bool HasBackedge,
2441 SmallPtrSetImpl<Value *> &Visited) {
2442 if (!okayForPHIOfOps(I))
2445 if (!Visited.insert(I).second)
2447 // For now, we require the instruction be cycle free because we don't
2448 // *always* create a phi of ops for instructions that could be done as phi
2449 // of ops, we only do it if we think it is useful. If we did do it all the
2450 // time, we could remove the cycle free check.
2451 if (!isCycleFree(I))
2454 unsigned IDFSNum = InstrToDFSNum(I);
2455 // Pretty much all of the instructions we can convert to phi of ops over a
2456 // backedge that are adds, are really induction variables, and those are
2457 // pretty much pointless to convert. This is very coarse-grained for a
2458 // test, so if we do find some value, we can change it later.
2459 // But otherwise, what can happen is we convert the induction variable from
2465 // i = phi (0, tmpphi)
2466 // tmpphi = phi(1, tmpphi+1)
2468 // Which we don't want to happen. We could just avoid this for all non-cycle
2469 // free phis, and we made go that route.
2470 if (HasBackedge && I->getOpcode() == Instruction::Add)
2473 SmallPtrSet<const Value *, 8> ProcessedPHIs;
2474 // TODO: We don't do phi translation on memory accesses because it's
2475 // complicated. For a load, we'd need to be able to simulate a new memoryuse,
2476 // which we don't have a good way of doing ATM.
2477 auto *MemAccess = getMemoryAccess(I);
2478 // If the memory operation is defined by a memory operation this block that
2479 // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi
2480 // can't help, as it would still be killed by that memory operation.
2481 if (MemAccess && !isa<MemoryPhi>(MemAccess->getDefiningAccess()) &&
2482 MemAccess->getDefiningAccess()->getBlock() == I->getParent())
2485 // Convert op of phis to phi of ops
2486 for (auto &Op : I->operands()) {
2487 if (!isa<PHINode>(Op))
2489 auto *OpPHI = cast<PHINode>(Op);
2490 // No point in doing this for one-operand phis.
2491 if (OpPHI->getNumOperands() == 1)
2493 if (!DebugCounter::shouldExecute(PHIOfOpsCounter))
2495 SmallVector<std::pair<Value *, BasicBlock *>, 4> Ops;
2496 auto *PHIBlock = getBlockForValue(OpPHI);
2497 for (auto PredBB : OpPHI->blocks()) {
2498 Value *FoundVal = nullptr;
2499 // We could just skip unreachable edges entirely but it's tricky to do
2500 // with rewriting existing phi nodes.
2501 if (ReachableEdges.count({PredBB, PHIBlock})) {
2502 // Clone the instruction, create an expression from it, and see if we
2504 Instruction *ValueOp = I->clone();
2505 auto Iter = TempToMemory.end();
2507 Iter = TempToMemory.insert({ValueOp, MemAccess}).first;
2509 for (auto &Op : ValueOp->operands()) {
2510 Op = Op->DoPHITranslation(PHIBlock, PredBB);
2511 // When this operand changes, it could change whether there is a
2512 // leader for us or not.
2513 addAdditionalUsers(Op, I);
2515 // Make sure it's marked as a temporary instruction.
2516 AllTempInstructions.insert(ValueOp);
2517 // and make sure anything that tries to add it's DFS number is
2518 // redirected to the instruction we are making a phi of ops
2520 InstrDFS.insert({ValueOp, IDFSNum});
2521 const Expression *E = performSymbolicEvaluation(ValueOp, Visited);
2522 InstrDFS.erase(ValueOp);
2523 AllTempInstructions.erase(ValueOp);
2524 ValueOp->deleteValue();
2526 TempToMemory.erase(Iter);
2529 FoundVal = findPhiOfOpsLeader(E, PredBB);
2531 ExpressionToPhiOfOps[E].insert(I);
2534 if (auto *SI = dyn_cast<StoreInst>(FoundVal))
2535 FoundVal = SI->getValueOperand();
2537 DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "
2538 << getBlockName(PredBB)
2539 << " because the block is unreachable\n");
2540 FoundVal = UndefValue::get(I->getType());
2543 Ops.push_back({FoundVal, PredBB});
2544 DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "
2545 << getBlockName(PredBB) << "\n");
2547 auto *ValuePHI = RealToTemp.lookup(I);
2548 bool NewPHI = false;
2550 ValuePHI = PHINode::Create(I->getType(), OpPHI->getNumOperands());
2551 addPhiOfOps(ValuePHI, PHIBlock, I);
2553 NumGVNPHIOfOpsCreated++;
2556 for (auto PHIOp : Ops)
2557 ValuePHI->addIncoming(PHIOp.first, PHIOp.second);
2560 for (auto PHIOp : Ops) {
2561 ValuePHI->setIncomingValue(i, PHIOp.first);
2562 ValuePHI->setIncomingBlock(i, PHIOp.second);
2567 DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *I
2569 return performSymbolicEvaluation(ValuePHI, Visited);
2574 // The algorithm initially places the values of the routine in the TOP
2575 // congruence class. The leader of TOP is the undetermined value `undef`.
2576 // When the algorithm has finished, values still in TOP are unreachable.
2577 void NewGVN::initializeCongruenceClasses(Function &F) {
2578 NextCongruenceNum = 0;
2580 // Note that even though we use the live on entry def as a representative
2581 // MemoryAccess, it is *not* the same as the actual live on entry def. We
2582 // have no real equivalemnt to undef for MemoryAccesses, and so we really
2583 // should be checking whether the MemoryAccess is top if we want to know if it
2584 // is equivalent to everything. Otherwise, what this really signifies is that
2585 // the access "it reaches all the way back to the beginning of the function"
2587 // Initialize all other instructions to be in TOP class.
2588 TOPClass = createCongruenceClass(nullptr, nullptr);
2589 TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());
2590 // The live on entry def gets put into it's own class
2591 MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =
2592 createMemoryClass(MSSA->getLiveOnEntryDef());
2594 for (auto DTN : nodes(DT)) {
2595 BasicBlock *BB = DTN->getBlock();
2596 // All MemoryAccesses are equivalent to live on entry to start. They must
2597 // be initialized to something so that initial changes are noticed. For
2598 // the maximal answer, we initialize them all to be the same as
2600 auto *MemoryBlockDefs = MSSA->getBlockDefs(BB);
2601 if (MemoryBlockDefs)
2602 for (const auto &Def : *MemoryBlockDefs) {
2603 MemoryAccessToClass[&Def] = TOPClass;
2604 auto *MD = dyn_cast<MemoryDef>(&Def);
2605 // Insert the memory phis into the member list.
2607 const MemoryPhi *MP = cast<MemoryPhi>(&Def);
2608 TOPClass->memory_insert(MP);
2609 MemoryPhiState.insert({MP, MPS_TOP});
2612 if (MD && isa<StoreInst>(MD->getMemoryInst()))
2613 TOPClass->incStoreCount();
2615 for (auto &I : *BB) {
2616 // TODO: Move to helper
2617 if (isa<PHINode>(&I))
2618 for (auto *U : I.users())
2619 if (auto *UInst = dyn_cast<Instruction>(U))
2620 if (InstrToDFSNum(UInst) != 0 && okayForPHIOfOps(UInst))
2621 PHINodeUses.insert(UInst);
2622 // Don't insert void terminators into the class. We don't value number
2623 // them, and they just end up sitting in TOP.
2624 if (isa<TerminatorInst>(I) && I.getType()->isVoidTy())
2626 TOPClass->insert(&I);
2627 ValueToClass[&I] = TOPClass;
2631 // Initialize arguments to be in their own unique congruence classes
2632 for (auto &FA : F.args())
2633 createSingletonCongruenceClass(&FA);
2636 void NewGVN::cleanupTables() {
2637 for (unsigned i = 0, e = CongruenceClasses.size(); i != e; ++i) {
2638 DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->getID()
2639 << " has " << CongruenceClasses[i]->size() << " members\n");
2640 // Make sure we delete the congruence class (probably worth switching to
2641 // a unique_ptr at some point.
2642 delete CongruenceClasses[i];
2643 CongruenceClasses[i] = nullptr;
2646 // Destroy the value expressions
2647 SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(),
2648 AllTempInstructions.end());
2649 AllTempInstructions.clear();
2651 // We have to drop all references for everything first, so there are no uses
2652 // left as we delete them.
2653 for (auto *I : TempInst) {
2654 I->dropAllReferences();
2657 while (!TempInst.empty()) {
2658 auto *I = TempInst.back();
2659 TempInst.pop_back();
2663 ValueToClass.clear();
2664 ArgRecycler.clear(ExpressionAllocator);
2665 ExpressionAllocator.Reset();
2666 CongruenceClasses.clear();
2667 ExpressionToClass.clear();
2668 ValueToExpression.clear();
2670 AdditionalUsers.clear();
2671 ExpressionToPhiOfOps.clear();
2672 TempToBlock.clear();
2673 TempToMemory.clear();
2674 PHIOfOpsPHIs.clear();
2675 ReachableBlocks.clear();
2676 ReachableEdges.clear();
2678 ProcessedCount.clear();
2681 InstructionsToErase.clear();
2683 BlockInstRange.clear();
2684 TouchedInstructions.clear();
2685 MemoryAccessToClass.clear();
2686 PredicateToUsers.clear();
2687 MemoryToUsers.clear();
2690 // Assign local DFS number mapping to instructions, and leave space for Value
2692 std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
2694 unsigned End = Start;
2695 if (MemoryAccess *MemPhi = getMemoryAccess(B)) {
2696 InstrDFS[MemPhi] = End++;
2697 DFSToInstr.emplace_back(MemPhi);
2700 // Then the real block goes next.
2701 for (auto &I : *B) {
2702 // There's no need to call isInstructionTriviallyDead more than once on
2703 // an instruction. Therefore, once we know that an instruction is dead
2704 // we change its DFS number so that it doesn't get value numbered.
2705 if (isInstructionTriviallyDead(&I, TLI)) {
2707 DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");
2708 markInstructionForDeletion(&I);
2711 InstrDFS[&I] = End++;
2712 DFSToInstr.emplace_back(&I);
2715 // All of the range functions taken half-open ranges (open on the end side).
2716 // So we do not subtract one from count, because at this point it is one
2717 // greater than the last instruction.
2718 return std::make_pair(Start, End);
2721 void NewGVN::updateProcessedCount(const Value *V) {
2723 if (ProcessedCount.count(V) == 0) {
2724 ProcessedCount.insert({V, 1});
2726 ++ProcessedCount[V];
2727 assert(ProcessedCount[V] < 100 &&
2728 "Seem to have processed the same Value a lot");
2732 // Evaluate MemoryPhi nodes symbolically, just like PHI nodes
2733 void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
2734 // If all the arguments are the same, the MemoryPhi has the same value as the
2735 // argument. Filter out unreachable blocks and self phis from our operands.
2736 // TODO: We could do cycle-checking on the memory phis to allow valueizing for
2737 // self-phi checking.
2738 const BasicBlock *PHIBlock = MP->getBlock();
2739 auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {
2740 return cast<MemoryAccess>(U) != MP &&
2741 !isMemoryAccessTOP(cast<MemoryAccess>(U)) &&
2742 ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock});
2744 // If all that is left is nothing, our memoryphi is undef. We keep it as
2745 // InitialClass. Note: The only case this should happen is if we have at
2746 // least one self-argument.
2747 if (Filtered.begin() == Filtered.end()) {
2748 if (setMemoryClass(MP, TOPClass))
2749 markMemoryUsersTouched(MP);
2753 // Transform the remaining operands into operand leaders.
2754 // FIXME: mapped_iterator should have a range version.
2755 auto LookupFunc = [&](const Use &U) {
2756 return lookupMemoryLeader(cast<MemoryAccess>(U));
2758 auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);
2759 auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);
2761 // and now check if all the elements are equal.
2762 // Sadly, we can't use std::equals since these are random access iterators.
2763 const auto *AllSameValue = *MappedBegin;
2765 bool AllEqual = std::all_of(
2766 MappedBegin, MappedEnd,
2767 [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });
2770 DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue << "\n");
2772 DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
2773 // If it's equal to something, it's in that class. Otherwise, it has to be in
2774 // a class where it is the leader (other things may be equivalent to it, but
2775 // it needs to start off in its own class, which means it must have been the
2776 // leader, and it can't have stopped being the leader because it was never
2778 CongruenceClass *CC =
2779 AllEqual ? getMemoryClass(AllSameValue) : ensureLeaderOfMemoryClass(MP);
2780 auto OldState = MemoryPhiState.lookup(MP);
2781 assert(OldState != MPS_Invalid && "Invalid memory phi state");
2782 auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique;
2783 MemoryPhiState[MP] = NewState;
2784 if (setMemoryClass(MP, CC) || OldState != NewState)
2785 markMemoryUsersTouched(MP);
2788 // Value number a single instruction, symbolically evaluating, performing
2789 // congruence finding, and updating mappings.
2790 void NewGVN::valueNumberInstruction(Instruction *I) {
2791 DEBUG(dbgs() << "Processing instruction " << *I << "\n");
2792 if (!I->isTerminator()) {
2793 const Expression *Symbolized = nullptr;
2794 SmallPtrSet<Value *, 2> Visited;
2795 if (DebugCounter::shouldExecute(VNCounter)) {
2796 Symbolized = performSymbolicEvaluation(I, Visited);
2797 // Make a phi of ops if necessary
2798 if (Symbolized && !isa<ConstantExpression>(Symbolized) &&
2799 !isa<VariableExpression>(Symbolized) && PHINodeUses.count(I)) {
2800 // FIXME: Backedge argument
2801 auto *PHIE = makePossiblePhiOfOps(I, false, Visited);
2807 // Mark the instruction as unused so we don't value number it again.
2810 // If we couldn't come up with a symbolic expression, use the unknown
2812 if (Symbolized == nullptr)
2813 Symbolized = createUnknownExpression(I);
2814 performCongruenceFinding(I, Symbolized);
2816 // Handle terminators that return values. All of them produce values we
2817 // don't currently understand. We don't place non-value producing
2818 // terminators in a class.
2819 if (!I->getType()->isVoidTy()) {
2820 auto *Symbolized = createUnknownExpression(I);
2821 performCongruenceFinding(I, Symbolized);
2823 processOutgoingEdges(dyn_cast<TerminatorInst>(I), I->getParent());
2827 // Check if there is a path, using single or equal argument phi nodes, from
2829 bool NewGVN::singleReachablePHIPath(
2830 SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First,
2831 const MemoryAccess *Second) const {
2832 if (First == Second)
2834 if (MSSA->isLiveOnEntryDef(First))
2837 // This is not perfect, but as we're just verifying here, we can live with
2838 // the loss of precision. The real solution would be that of doing strongly
2839 // connected component finding in this routine, and it's probably not worth
2840 // the complexity for the time being. So, we just keep a set of visited
2841 // MemoryAccess and return true when we hit a cycle.
2842 if (Visited.count(First))
2844 Visited.insert(First);
2846 const auto *EndDef = First;
2847 for (auto *ChainDef : optimized_def_chain(First)) {
2848 if (ChainDef == Second)
2850 if (MSSA->isLiveOnEntryDef(ChainDef))
2854 auto *MP = cast<MemoryPhi>(EndDef);
2855 auto ReachableOperandPred = [&](const Use &U) {
2856 return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()});
2858 auto FilteredPhiArgs =
2859 make_filter_range(MP->operands(), ReachableOperandPred);
2860 SmallVector<const Value *, 32> OperandList;
2861 std::copy(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
2862 std::back_inserter(OperandList));
2863 bool Okay = OperandList.size() == 1;
2866 std::equal(OperandList.begin(), OperandList.end(), OperandList.begin());
2868 return singleReachablePHIPath(Visited, cast<MemoryAccess>(OperandList[0]),
2873 // Verify the that the memory equivalence table makes sense relative to the
2874 // congruence classes. Note that this checking is not perfect, and is currently
2875 // subject to very rare false negatives. It is only useful for
2876 // testing/debugging.
2877 void NewGVN::verifyMemoryCongruency() const {
2879 // Verify that the memory table equivalence and memory member set match
2880 for (const auto *CC : CongruenceClasses) {
2881 if (CC == TOPClass || CC->isDead())
2883 if (CC->getStoreCount() != 0) {
2884 assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&
2885 "Any class with a store as a leader should have a "
2886 "representative stored value");
2887 assert(CC->getMemoryLeader() &&
2888 "Any congruence class with a store should have a "
2889 "representative access");
2892 if (CC->getMemoryLeader())
2893 assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&
2894 "Representative MemoryAccess does not appear to be reverse "
2896 for (auto M : CC->memory())
2897 assert(MemoryAccessToClass.lookup(M) == CC &&
2898 "Memory member does not appear to be reverse mapped properly");
2901 // Anything equivalent in the MemoryAccess table should be in the same
2902 // congruence class.
2904 // Filter out the unreachable and trivially dead entries, because they may
2905 // never have been updated if the instructions were not processed.
2906 auto ReachableAccessPred =
2907 [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {
2908 bool Result = ReachableBlocks.count(Pair.first->getBlock());
2909 if (!Result || MSSA->isLiveOnEntryDef(Pair.first) ||
2910 MemoryToDFSNum(Pair.first) == 0)
2912 if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
2913 return !isInstructionTriviallyDead(MemDef->getMemoryInst());
2915 // We could have phi nodes which operands are all trivially dead,
2916 // so we don't process them.
2917 if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) {
2918 for (auto &U : MemPHI->incoming_values()) {
2919 if (Instruction *I = dyn_cast<Instruction>(U.get())) {
2920 if (!isInstructionTriviallyDead(I))
2930 auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);
2931 for (auto KV : Filtered) {
2932 if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
2933 auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());
2934 if (FirstMUD && SecondMUD) {
2935 SmallPtrSet<const MemoryAccess *, 8> VisitedMAS;
2936 assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||
2937 ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
2938 ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
2939 "The instructions for these memory operations should have "
2940 "been in the same congruence class or reachable through"
2941 "a single argument phi");
2943 } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
2944 // We can only sanely verify that MemoryDefs in the operand list all have
2946 auto ReachableOperandPred = [&](const Use &U) {
2947 return ReachableEdges.count(
2948 {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&
2952 // All arguments should in the same class, ignoring unreachable arguments
2953 auto FilteredPhiArgs =
2954 make_filter_range(FirstMP->operands(), ReachableOperandPred);
2955 SmallVector<const CongruenceClass *, 16> PhiOpClasses;
2956 std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
2957 std::back_inserter(PhiOpClasses), [&](const Use &U) {
2958 const MemoryDef *MD = cast<MemoryDef>(U);
2959 return ValueToClass.lookup(MD->getMemoryInst());
2961 assert(std::equal(PhiOpClasses.begin(), PhiOpClasses.end(),
2962 PhiOpClasses.begin()) &&
2963 "All MemoryPhi arguments should be in the same class");
2969 // Verify that the sparse propagation we did actually found the maximal fixpoint
2970 // We do this by storing the value to class mapping, touching all instructions,
2971 // and redoing the iteration to see if anything changed.
2972 void NewGVN::verifyIterationSettled(Function &F) {
2974 DEBUG(dbgs() << "Beginning iteration verification\n");
2975 if (DebugCounter::isCounterSet(VNCounter))
2976 DebugCounter::setCounterValue(VNCounter, StartingVNCounter);
2978 // Note that we have to store the actual classes, as we may change existing
2979 // classes during iteration. This is because our memory iteration propagation
2980 // is not perfect, and so may waste a little work. But it should generate
2981 // exactly the same congruence classes we have now, with different IDs.
2982 std::map<const Value *, CongruenceClass> BeforeIteration;
2984 for (auto &KV : ValueToClass) {
2985 if (auto *I = dyn_cast<Instruction>(KV.first))
2986 // Skip unused/dead instructions.
2987 if (InstrToDFSNum(I) == 0)
2989 BeforeIteration.insert({KV.first, *KV.second});
2992 TouchedInstructions.set();
2993 TouchedInstructions.reset(0);
2994 iterateTouchedInstructions();
2995 DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>>
2997 for (const auto &KV : ValueToClass) {
2998 if (auto *I = dyn_cast<Instruction>(KV.first))
2999 // Skip unused/dead instructions.
3000 if (InstrToDFSNum(I) == 0)
3002 // We could sink these uses, but i think this adds a bit of clarity here as
3003 // to what we are comparing.
3004 auto *BeforeCC = &BeforeIteration.find(KV.first)->second;
3005 auto *AfterCC = KV.second;
3006 // Note that the classes can't change at this point, so we memoize the set
3008 if (!EqualClasses.count({BeforeCC, AfterCC})) {
3009 assert(BeforeCC->isEquivalentTo(AfterCC) &&
3010 "Value number changed after main loop completed!");
3011 EqualClasses.insert({BeforeCC, AfterCC});
3017 // Verify that for each store expression in the expression to class mapping,
3018 // only the latest appears, and multiple ones do not appear.
3019 // Because loads do not use the stored value when doing equality with stores,
3020 // if we don't erase the old store expressions from the table, a load can find
3021 // a no-longer valid StoreExpression.
3022 void NewGVN::verifyStoreExpressions() const {
3024 DenseSet<std::pair<const Value *, const Value *>> StoreExpressionSet;
3025 for (const auto &KV : ExpressionToClass) {
3026 if (auto *SE = dyn_cast<StoreExpression>(KV.first)) {
3027 // Make sure a version that will conflict with loads is not already there
3029 StoreExpressionSet.insert({SE->getOperand(0), SE->getMemoryLeader()});
3030 assert(Res.second &&
3031 "Stored expression conflict exists in expression table");
3032 auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst());
3033 assert(ValueExpr && ValueExpr->equals(*SE) &&
3034 "StoreExpression in ExpressionToClass is not latest "
3035 "StoreExpression for value");
3041 // This is the main value numbering loop, it iterates over the initial touched
3042 // instruction set, propagating value numbers, marking things touched, etc,
3043 // until the set of touched instructions is completely empty.
3044 void NewGVN::iterateTouchedInstructions() {
3045 unsigned int Iterations = 0;
3046 // Figure out where touchedinstructions starts
3047 int FirstInstr = TouchedInstructions.find_first();
3048 // Nothing set, nothing to iterate, just return.
3049 if (FirstInstr == -1)
3051 const BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr));
3052 while (TouchedInstructions.any()) {
3054 // Walk through all the instructions in all the blocks in RPO.
3055 // TODO: As we hit a new block, we should push and pop equalities into a
3056 // table lookupOperandLeader can use, to catch things PredicateInfo
3057 // might miss, like edge-only equivalences.
3058 for (unsigned InstrNum : TouchedInstructions.set_bits()) {
3060 // This instruction was found to be dead. We don't bother looking
3062 if (InstrNum == 0) {
3063 TouchedInstructions.reset(InstrNum);
3067 Value *V = InstrFromDFSNum(InstrNum);
3068 const BasicBlock *CurrBlock = getBlockForValue(V);
3070 // If we hit a new block, do reachability processing.
3071 if (CurrBlock != LastBlock) {
3072 LastBlock = CurrBlock;
3073 bool BlockReachable = ReachableBlocks.count(CurrBlock);
3074 const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);
3076 // If it's not reachable, erase any touched instructions and move on.
3077 if (!BlockReachable) {
3078 TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);
3079 DEBUG(dbgs() << "Skipping instructions in block "
3080 << getBlockName(CurrBlock)
3081 << " because it is unreachable\n");
3084 updateProcessedCount(CurrBlock);
3087 if (auto *MP = dyn_cast<MemoryPhi>(V)) {
3088 DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
3089 valueNumberMemoryPhi(MP);
3090 } else if (auto *I = dyn_cast<Instruction>(V)) {
3091 valueNumberInstruction(I);
3093 llvm_unreachable("Should have been a MemoryPhi or Instruction");
3095 updateProcessedCount(V);
3096 // Reset after processing (because we may mark ourselves as touched when
3097 // we propagate equalities).
3098 TouchedInstructions.reset(InstrNum);
3101 NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);
3104 // This is the main transformation entry point.
3105 bool NewGVN::runGVN() {
3106 if (DebugCounter::isCounterSet(VNCounter))
3107 StartingVNCounter = DebugCounter::getCounterValue(VNCounter);
3108 bool Changed = false;
3109 NumFuncArgs = F.arg_size();
3110 MSSAWalker = MSSA->getWalker();
3111 SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();
3113 // Count number of instructions for sizing of hash tables, and come
3114 // up with a global dfs numbering for instructions.
3115 unsigned ICount = 1;
3116 // Add an empty instruction to account for the fact that we start at 1
3117 DFSToInstr.emplace_back(nullptr);
3118 // Note: We want ideal RPO traversal of the blocks, which is not quite the
3119 // same as dominator tree order, particularly with regard whether backedges
3120 // get visited first or second, given a block with multiple successors.
3121 // If we visit in the wrong order, we will end up performing N times as many
3123 // The dominator tree does guarantee that, for a given dom tree node, it's
3124 // parent must occur before it in the RPO ordering. Thus, we only need to sort
3126 ReversePostOrderTraversal<Function *> RPOT(&F);
3127 unsigned Counter = 0;
3128 for (auto &B : RPOT) {
3129 auto *Node = DT->getNode(B);
3130 assert(Node && "RPO and Dominator tree should have same reachability");
3131 RPOOrdering[Node] = ++Counter;
3133 // Sort dominator tree children arrays into RPO.
3134 for (auto &B : RPOT) {
3135 auto *Node = DT->getNode(B);
3136 if (Node->getChildren().size() > 1)
3137 std::sort(Node->begin(), Node->end(),
3138 [&](const DomTreeNode *A, const DomTreeNode *B) {
3139 return RPOOrdering[A] < RPOOrdering[B];
3143 // Now a standard depth first ordering of the domtree is equivalent to RPO.
3144 for (auto DTN : depth_first(DT->getRootNode())) {
3145 BasicBlock *B = DTN->getBlock();
3146 const auto &BlockRange = assignDFSNumbers(B, ICount);
3147 BlockInstRange.insert({B, BlockRange});
3148 ICount += BlockRange.second - BlockRange.first;
3150 initializeCongruenceClasses(F);
3152 TouchedInstructions.resize(ICount);
3153 // Ensure we don't end up resizing the expressionToClass map, as
3154 // that can be quite expensive. At most, we have one expression per
3156 ExpressionToClass.reserve(ICount);
3158 // Initialize the touched instructions to include the entry block.
3159 const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());
3160 TouchedInstructions.set(InstRange.first, InstRange.second);
3161 DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())
3162 << " marked reachable\n");
3163 ReachableBlocks.insert(&F.getEntryBlock());
3165 iterateTouchedInstructions();
3166 verifyMemoryCongruency();
3167 verifyIterationSettled(F);
3168 verifyStoreExpressions();
3170 Changed |= eliminateInstructions(F);
3172 // Delete all instructions marked for deletion.
3173 for (Instruction *ToErase : InstructionsToErase) {
3174 if (!ToErase->use_empty())
3175 ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType()));
3177 if (ToErase->getParent())
3178 ToErase->eraseFromParent();
3181 // Delete all unreachable blocks.
3182 auto UnreachableBlockPred = [&](const BasicBlock &BB) {
3183 return !ReachableBlocks.count(&BB);
3186 for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {
3187 DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
3188 << " is unreachable\n");
3189 deleteInstructionsInBlock(&BB);
3197 struct NewGVN::ValueDFS {
3201 // Only one of Def and U will be set.
3202 // The bool in the Def tells us whether the Def is the stored value of a
3204 PointerIntPair<Value *, 1, bool> Def;
3206 bool operator<(const ValueDFS &Other) const {
3207 // It's not enough that any given field be less than - we have sets
3208 // of fields that need to be evaluated together to give a proper ordering.
3209 // For example, if you have;
3214 // We want the second to be less than the first, but if we just go field
3215 // by field, we will get to Val 0 < Val 50 and say the first is less than
3216 // the second. We only want it to be less than if the DFS orders are equal.
3218 // Each LLVM instruction only produces one value, and thus the lowest-level
3219 // differentiator that really matters for the stack (and what we use as as a
3220 // replacement) is the local dfs number.
3221 // Everything else in the structure is instruction level, and only affects
3222 // the order in which we will replace operands of a given instruction.
3224 // For a given instruction (IE things with equal dfsin, dfsout, localnum),
3225 // the order of replacement of uses does not matter.
3229 // When you hit b, you will have two valuedfs with the same dfsin, out, and
3231 // The .val will be the same as well.
3232 // The .u's will be different.
3233 // You will replace both, and it does not matter what order you replace them
3234 // in (IE whether you replace operand 2, then operand 1, or operand 1, then
3236 // Similarly for the case of same dfsin, dfsout, localnum, but different
3241 // in c, we will a valuedfs for a, and one for b,with everything the same
3243 // It does not matter what order we replace these operands in.
3244 // You will always end up with the same IR, and this is guaranteed.
3245 return std::tie(DFSIn, DFSOut, LocalNum, Def, U) <
3246 std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def,
3251 // This function converts the set of members for a congruence class from values,
3252 // to sets of defs and uses with associated DFS info. The total number of
3253 // reachable uses for each value is stored in UseCount, and instructions that
3255 // dead (have no non-dead uses) are stored in ProbablyDead.
3256 void NewGVN::convertClassToDFSOrdered(
3257 const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,
3258 DenseMap<const Value *, unsigned int> &UseCounts,
3259 SmallPtrSetImpl<Instruction *> &ProbablyDead) const {
3260 for (auto D : Dense) {
3261 // First add the value.
3262 BasicBlock *BB = getBlockForValue(D);
3263 // Constants are handled prior to ever calling this function, so
3264 // we should only be left with instructions as members.
3265 assert(BB && "Should have figured out a basic block for value");
3267 DomTreeNode *DomNode = DT->getNode(BB);
3268 VDDef.DFSIn = DomNode->getDFSNumIn();
3269 VDDef.DFSOut = DomNode->getDFSNumOut();
3270 // If it's a store, use the leader of the value operand, if it's always
3271 // available, or the value operand. TODO: We could do dominance checks to
3272 // find a dominating leader, but not worth it ATM.
3273 if (auto *SI = dyn_cast<StoreInst>(D)) {
3274 auto Leader = lookupOperandLeader(SI->getValueOperand());
3275 if (alwaysAvailable(Leader)) {
3276 VDDef.Def.setPointer(Leader);
3278 VDDef.Def.setPointer(SI->getValueOperand());
3279 VDDef.Def.setInt(true);
3282 VDDef.Def.setPointer(D);
3284 assert(isa<Instruction>(D) &&
3285 "The dense set member should always be an instruction");
3286 Instruction *Def = cast<Instruction>(D);
3287 VDDef.LocalNum = InstrToDFSNum(D);
3288 DFSOrderedSet.push_back(VDDef);
3289 // If there is a phi node equivalent, add it
3290 if (auto *PN = RealToTemp.lookup(Def)) {
3292 dyn_cast_or_null<PHIExpression>(ValueToExpression.lookup(Def));
3294 VDDef.Def.setInt(false);
3295 VDDef.Def.setPointer(PN);
3297 DFSOrderedSet.push_back(VDDef);
3301 unsigned int UseCount = 0;
3302 // Now add the uses.
3303 for (auto &U : Def->uses()) {
3304 if (auto *I = dyn_cast<Instruction>(U.getUser())) {
3305 // Don't try to replace into dead uses
3306 if (InstructionsToErase.count(I))
3309 // Put the phi node uses in the incoming block.
3311 if (auto *P = dyn_cast<PHINode>(I)) {
3312 IBlock = P->getIncomingBlock(U);
3313 // Make phi node users appear last in the incoming block
3315 VDUse.LocalNum = InstrDFS.size() + 1;
3317 IBlock = getBlockForValue(I);
3318 VDUse.LocalNum = InstrToDFSNum(I);
3321 // Skip uses in unreachable blocks, as we're going
3323 if (ReachableBlocks.count(IBlock) == 0)
3326 DomTreeNode *DomNode = DT->getNode(IBlock);
3327 VDUse.DFSIn = DomNode->getDFSNumIn();
3328 VDUse.DFSOut = DomNode->getDFSNumOut();
3331 DFSOrderedSet.emplace_back(VDUse);
3335 // If there are no uses, it's probably dead (but it may have side-effects,
3336 // so not definitely dead. Otherwise, store the number of uses so we can
3337 // track if it becomes dead later).
3339 ProbablyDead.insert(Def);
3341 UseCounts[Def] = UseCount;
3345 // This function converts the set of members for a congruence class from values,
3346 // to the set of defs for loads and stores, with associated DFS info.
3347 void NewGVN::convertClassToLoadsAndStores(
3348 const CongruenceClass &Dense,
3349 SmallVectorImpl<ValueDFS> &LoadsAndStores) const {
3350 for (auto D : Dense) {
3351 if (!isa<LoadInst>(D) && !isa<StoreInst>(D))
3354 BasicBlock *BB = getBlockForValue(D);
3356 DomTreeNode *DomNode = DT->getNode(BB);
3357 VD.DFSIn = DomNode->getDFSNumIn();
3358 VD.DFSOut = DomNode->getDFSNumOut();
3359 VD.Def.setPointer(D);
3361 // If it's an instruction, use the real local dfs number.
3362 if (auto *I = dyn_cast<Instruction>(D))
3363 VD.LocalNum = InstrToDFSNum(I);
3365 llvm_unreachable("Should have been an instruction");
3367 LoadsAndStores.emplace_back(VD);
3371 static void patchReplacementInstruction(Instruction *I, Value *Repl) {
3372 auto *ReplInst = dyn_cast<Instruction>(Repl);
3376 // Patch the replacement so that it is not more restrictive than the value
3378 // Note that if 'I' is a load being replaced by some operation,
3379 // for example, by an arithmetic operation, then andIRFlags()
3380 // would just erase all math flags from the original arithmetic
3381 // operation, which is clearly not wanted and not needed.
3382 if (!isa<LoadInst>(I))
3383 ReplInst->andIRFlags(I);
3385 // FIXME: If both the original and replacement value are part of the
3386 // same control-flow region (meaning that the execution of one
3387 // guarantees the execution of the other), then we can combine the
3388 // noalias scopes here and do better than the general conservative
3389 // answer used in combineMetadata().
3391 // In general, GVN unifies expressions over different control-flow
3392 // regions, and so we need a conservative combination of the noalias
3394 static const unsigned KnownIDs[] = {
3395 LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
3396 LLVMContext::MD_noalias, LLVMContext::MD_range,
3397 LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load,
3398 LLVMContext::MD_invariant_group};
3399 combineMetadata(ReplInst, I, KnownIDs);
3402 static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
3403 patchReplacementInstruction(I, Repl);
3404 I->replaceAllUsesWith(Repl);
3407 void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
3408 DEBUG(dbgs() << " BasicBlock Dead:" << *BB);
3409 ++NumGVNBlocksDeleted;
3411 // Delete the instructions backwards, as it has a reduced likelihood of having
3412 // to update as many def-use and use-def chains. Start after the terminator.
3413 auto StartPoint = BB->rbegin();
3415 // Note that we explicitly recalculate BB->rend() on each iteration,
3416 // as it may change when we remove the first instruction.
3417 for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
3418 Instruction &Inst = *I++;
3419 if (!Inst.use_empty())
3420 Inst.replaceAllUsesWith(UndefValue::get(Inst.getType()));
3421 if (isa<LandingPadInst>(Inst))
3424 Inst.eraseFromParent();
3425 ++NumGVNInstrDeleted;
3427 // Now insert something that simplifycfg will turn into an unreachable.
3428 Type *Int8Ty = Type::getInt8Ty(BB->getContext());
3429 new StoreInst(UndefValue::get(Int8Ty),
3430 Constant::getNullValue(Int8Ty->getPointerTo()),
3431 BB->getTerminator());
3434 void NewGVN::markInstructionForDeletion(Instruction *I) {
3435 DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
3436 InstructionsToErase.insert(I);
3439 void NewGVN::replaceInstruction(Instruction *I, Value *V) {
3441 DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
3442 patchAndReplaceAllUsesWith(I, V);
3443 // We save the actual erasing to avoid invalidating memory
3444 // dependencies until we are done with everything.
3445 markInstructionForDeletion(I);
3450 // This is a stack that contains both the value and dfs info of where
3451 // that value is valid.
3452 class ValueDFSStack {
3454 Value *back() const { return ValueStack.back(); }
3455 std::pair<int, int> dfs_back() const { return DFSStack.back(); }
3457 void push_back(Value *V, int DFSIn, int DFSOut) {
3458 ValueStack.emplace_back(V);
3459 DFSStack.emplace_back(DFSIn, DFSOut);
3461 bool empty() const { return DFSStack.empty(); }
3462 bool isInScope(int DFSIn, int DFSOut) const {
3465 return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
3468 void popUntilDFSScope(int DFSIn, int DFSOut) {
3470 // These two should always be in sync at this point.
3471 assert(ValueStack.size() == DFSStack.size() &&
3472 "Mismatch between ValueStack and DFSStack");
3474 !DFSStack.empty() &&
3475 !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {
3476 DFSStack.pop_back();
3477 ValueStack.pop_back();
3482 SmallVector<Value *, 8> ValueStack;
3483 SmallVector<std::pair<int, int>, 8> DFSStack;
3487 // Given a value and a basic block we are trying to see if it is available in,
3488 // see if the value has a leader available in that block.
3489 Value *NewGVN::findPhiOfOpsLeader(const Expression *E,
3490 const BasicBlock *BB) const {
3491 // It would already be constant if we could make it constant
3492 if (auto *CE = dyn_cast<ConstantExpression>(E))
3493 return CE->getConstantValue();
3494 if (auto *VE = dyn_cast<VariableExpression>(E))
3495 return VE->getVariableValue();
3497 auto *CC = ExpressionToClass.lookup(E);
3500 if (alwaysAvailable(CC->getLeader()))
3501 return CC->getLeader();
3503 for (auto Member : *CC) {
3504 auto *MemberInst = dyn_cast<Instruction>(Member);
3505 // Anything that isn't an instruction is always available.
3508 // If we are looking for something in the same block as the member, it must
3509 // be a leader because this function is looking for operands for a phi node.
3510 if (MemberInst->getParent() == BB ||
3511 DT->dominates(MemberInst->getParent(), BB)) {
3518 bool NewGVN::eliminateInstructions(Function &F) {
3519 // This is a non-standard eliminator. The normal way to eliminate is
3520 // to walk the dominator tree in order, keeping track of available
3521 // values, and eliminating them. However, this is mildly
3522 // pointless. It requires doing lookups on every instruction,
3523 // regardless of whether we will ever eliminate it. For
3524 // instructions part of most singleton congruence classes, we know we
3525 // will never eliminate them.
3527 // Instead, this eliminator looks at the congruence classes directly, sorts
3528 // them into a DFS ordering of the dominator tree, and then we just
3529 // perform elimination straight on the sets by walking the congruence
3530 // class member uses in order, and eliminate the ones dominated by the
3531 // last member. This is worst case O(E log E) where E = number of
3532 // instructions in a single congruence class. In theory, this is all
3533 // instructions. In practice, it is much faster, as most instructions are
3534 // either in singleton congruence classes or can't possibly be eliminated
3535 // anyway (if there are no overlapping DFS ranges in class).
3536 // When we find something not dominated, it becomes the new leader
3537 // for elimination purposes.
3538 // TODO: If we wanted to be faster, We could remove any members with no
3539 // overlapping ranges while sorting, as we will never eliminate anything
3540 // with those members, as they don't dominate anything else in our set.
3542 bool AnythingReplaced = false;
3544 // Since we are going to walk the domtree anyway, and we can't guarantee the
3545 // DFS numbers are updated, we compute some ourselves.
3546 DT->updateDFSNumbers();
3548 // Go through all of our phi nodes, and kill the arguments associated with
3549 // unreachable edges.
3550 auto ReplaceUnreachablePHIArgs = [&](PHINode &PHI, BasicBlock *BB) {
3551 for (auto &Operand : PHI.incoming_values())
3552 if (!ReachableEdges.count({PHI.getIncomingBlock(Operand), BB})) {
3553 DEBUG(dbgs() << "Replacing incoming value of " << PHI << " for block "
3554 << getBlockName(PHI.getIncomingBlock(Operand))
3555 << " with undef due to it being unreachable\n");
3556 Operand.set(UndefValue::get(PHI.getType()));
3559 SmallPtrSet<BasicBlock *, 8> BlocksWithPhis;
3561 if ((!B.empty() && isa<PHINode>(*B.begin())) ||
3562 (PHIOfOpsPHIs.find(&B) != PHIOfOpsPHIs.end()))
3563 BlocksWithPhis.insert(&B);
3564 DenseMap<const BasicBlock *, unsigned> ReachablePredCount;
3565 for (auto KV : ReachableEdges)
3566 ReachablePredCount[KV.getEnd()]++;
3567 for (auto *BB : BlocksWithPhis)
3568 // TODO: It would be faster to use getNumIncomingBlocks() on a phi node in
3569 // the block and subtract the pred count, but it's more complicated.
3570 if (ReachablePredCount.lookup(BB) !=
3571 std::distance(pred_begin(BB), pred_end(BB))) {
3572 for (auto II = BB->begin(); isa<PHINode>(II); ++II) {
3573 auto &PHI = cast<PHINode>(*II);
3574 ReplaceUnreachablePHIArgs(PHI, BB);
3576 for_each_found(PHIOfOpsPHIs, BB, [&](PHINode *PHI) {
3577 ReplaceUnreachablePHIArgs(*PHI, BB);
3581 // Map to store the use counts
3582 DenseMap<const Value *, unsigned int> UseCounts;
3583 for (auto *CC : reverse(CongruenceClasses)) {
3584 DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID() << "\n");
3585 // Track the equivalent store info so we can decide whether to try
3586 // dead store elimination.
3587 SmallVector<ValueDFS, 8> PossibleDeadStores;
3588 SmallPtrSet<Instruction *, 8> ProbablyDead;
3589 if (CC->isDead() || CC->empty())
3591 // Everything still in the TOP class is unreachable or dead.
3592 if (CC == TOPClass) {
3593 for (auto M : *CC) {
3594 auto *VTE = ValueToExpression.lookup(M);
3595 if (VTE && isa<DeadExpression>(VTE))
3596 markInstructionForDeletion(cast<Instruction>(M));
3597 assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||
3598 InstructionsToErase.count(cast<Instruction>(M))) &&
3599 "Everything in TOP should be unreachable or dead at this "
3605 assert(CC->getLeader() && "We should have had a leader");
3606 // If this is a leader that is always available, and it's a
3607 // constant or has no equivalences, just replace everything with
3608 // it. We then update the congruence class with whatever members
3611 CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
3612 if (alwaysAvailable(Leader)) {
3613 CongruenceClass::MemberSet MembersLeft;
3614 for (auto M : *CC) {
3616 // Void things have no uses we can replace.
3617 if (Member == Leader || !isa<Instruction>(Member) ||
3618 Member->getType()->isVoidTy()) {
3619 MembersLeft.insert(Member);
3622 DEBUG(dbgs() << "Found replacement " << *(Leader) << " for " << *Member
3624 auto *I = cast<Instruction>(Member);
3625 assert(Leader != I && "About to accidentally remove our leader");
3626 replaceInstruction(I, Leader);
3627 AnythingReplaced = true;
3629 CC->swap(MembersLeft);
3631 // If this is a singleton, we can skip it.
3632 if (CC->size() != 1 || RealToTemp.lookup(Leader)) {
3633 // This is a stack because equality replacement/etc may place
3634 // constants in the middle of the member list, and we want to use
3635 // those constant values in preference to the current leader, over
3636 // the scope of those constants.
3637 ValueDFSStack EliminationStack;
3639 // Convert the members to DFS ordered sets and then merge them.
3640 SmallVector<ValueDFS, 8> DFSOrderedSet;
3641 convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead);
3643 // Sort the whole thing.
3644 std::sort(DFSOrderedSet.begin(), DFSOrderedSet.end());
3645 for (auto &VD : DFSOrderedSet) {
3646 int MemberDFSIn = VD.DFSIn;
3647 int MemberDFSOut = VD.DFSOut;
3648 Value *Def = VD.Def.getPointer();
3649 bool FromStore = VD.Def.getInt();
3651 // We ignore void things because we can't get a value from them.
3652 if (Def && Def->getType()->isVoidTy())
3654 auto *DefInst = dyn_cast_or_null<Instruction>(Def);
3655 if (DefInst && AllTempInstructions.count(DefInst)) {
3656 auto *PN = cast<PHINode>(DefInst);
3658 // If this is a value phi and that's the expression we used, insert
3659 // it into the program
3660 // remove from temp instruction list.
3661 AllTempInstructions.erase(PN);
3662 auto *DefBlock = getBlockForValue(Def);
3663 DEBUG(dbgs() << "Inserting fully real phi of ops" << *Def
3665 << getBlockName(getBlockForValue(Def)) << "\n");
3666 PN->insertBefore(&DefBlock->front());
3668 NumGVNPHIOfOpsEliminations++;
3671 if (EliminationStack.empty()) {
3672 DEBUG(dbgs() << "Elimination Stack is empty\n");
3674 DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
3675 << EliminationStack.dfs_back().first << ","
3676 << EliminationStack.dfs_back().second << ")\n");
3679 DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
3680 << MemberDFSOut << ")\n");
3681 // First, we see if we are out of scope or empty. If so,
3682 // and there equivalences, we try to replace the top of
3683 // stack with equivalences (if it's on the stack, it must
3684 // not have been eliminated yet).
3685 // Then we synchronize to our current scope, by
3686 // popping until we are back within a DFS scope that
3687 // dominates the current member.
3688 // Then, what happens depends on a few factors
3689 // If the stack is now empty, we need to push
3690 // If we have a constant or a local equivalence we want to
3691 // start using, we also push.
3692 // Otherwise, we walk along, processing members who are
3693 // dominated by this scope, and eliminate them.
3694 bool ShouldPush = Def && EliminationStack.empty();
3696 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);
3698 if (OutOfScope || ShouldPush) {
3699 // Sync to our current scope.
3700 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
3701 bool ShouldPush = Def && EliminationStack.empty();
3703 EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut);
3707 // Skip the Def's, we only want to eliminate on their uses. But mark
3708 // dominated defs as dead.
3710 // For anything in this case, what and how we value number
3711 // guarantees that any side-effets that would have occurred (ie
3712 // throwing, etc) can be proven to either still occur (because it's
3713 // dominated by something that has the same side-effects), or never
3714 // occur. Otherwise, we would not have been able to prove it value
3715 // equivalent to something else. For these things, we can just mark
3716 // it all dead. Note that this is different from the "ProbablyDead"
3717 // set, which may not be dominated by anything, and thus, are only
3718 // easy to prove dead if they are also side-effect free. Note that
3719 // because stores are put in terms of the stored value, we skip
3720 // stored values here. If the stored value is really dead, it will
3721 // still be marked for deletion when we process it in its own class.
3722 if (!EliminationStack.empty() && Def != EliminationStack.back() &&
3723 isa<Instruction>(Def) && !FromStore)
3724 markInstructionForDeletion(cast<Instruction>(Def));
3727 // At this point, we know it is a Use we are trying to possibly
3730 assert(isa<Instruction>(U->get()) &&
3731 "Current def should have been an instruction");
3732 assert(isa<Instruction>(U->getUser()) &&
3733 "Current user should have been an instruction");
3735 // If the thing we are replacing into is already marked to be dead,
3736 // this use is dead. Note that this is true regardless of whether
3737 // we have anything dominating the use or not. We do this here
3738 // because we are already walking all the uses anyway.
3739 Instruction *InstUse = cast<Instruction>(U->getUser());
3740 if (InstructionsToErase.count(InstUse)) {
3741 auto &UseCount = UseCounts[U->get()];
3742 if (--UseCount == 0) {
3743 ProbablyDead.insert(cast<Instruction>(U->get()));
3747 // If we get to this point, and the stack is empty we must have a use
3748 // with nothing we can use to eliminate this use, so just skip it.
3749 if (EliminationStack.empty())
3752 Value *DominatingLeader = EliminationStack.back();
3754 auto *II = dyn_cast<IntrinsicInst>(DominatingLeader);
3755 if (II && II->getIntrinsicID() == Intrinsic::ssa_copy)
3756 DominatingLeader = II->getOperand(0);
3758 // Don't replace our existing users with ourselves.
3759 if (U->get() == DominatingLeader)
3761 DEBUG(dbgs() << "Found replacement " << *DominatingLeader << " for "
3762 << *U->get() << " in " << *(U->getUser()) << "\n");
3764 // If we replaced something in an instruction, handle the patching of
3765 // metadata. Skip this if we are replacing predicateinfo with its
3766 // original operand, as we already know we can just drop it.
3767 auto *ReplacedInst = cast<Instruction>(U->get());
3768 auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst);
3769 if (!PI || DominatingLeader != PI->OriginalOp)
3770 patchReplacementInstruction(ReplacedInst, DominatingLeader);
3771 U->set(DominatingLeader);
3772 // This is now a use of the dominating leader, which means if the
3773 // dominating leader was dead, it's now live!
3774 auto &LeaderUseCount = UseCounts[DominatingLeader];
3775 // It's about to be alive again.
3776 if (LeaderUseCount == 0 && isa<Instruction>(DominatingLeader))
3777 ProbablyDead.erase(cast<Instruction>(DominatingLeader));
3778 if (LeaderUseCount == 0 && II)
3779 ProbablyDead.insert(II);
3781 AnythingReplaced = true;
3786 // At this point, anything still in the ProbablyDead set is actually dead if
3787 // would be trivially dead.
3788 for (auto *I : ProbablyDead)
3789 if (wouldInstructionBeTriviallyDead(I))
3790 markInstructionForDeletion(I);
3792 // Cleanup the congruence class.
3793 CongruenceClass::MemberSet MembersLeft;
3794 for (auto *Member : *CC)
3795 if (!isa<Instruction>(Member) ||
3796 !InstructionsToErase.count(cast<Instruction>(Member)))
3797 MembersLeft.insert(Member);
3798 CC->swap(MembersLeft);
3800 // If we have possible dead stores to look at, try to eliminate them.
3801 if (CC->getStoreCount() > 0) {
3802 convertClassToLoadsAndStores(*CC, PossibleDeadStores);
3803 std::sort(PossibleDeadStores.begin(), PossibleDeadStores.end());
3804 ValueDFSStack EliminationStack;
3805 for (auto &VD : PossibleDeadStores) {
3806 int MemberDFSIn = VD.DFSIn;
3807 int MemberDFSOut = VD.DFSOut;
3808 Instruction *Member = cast<Instruction>(VD.Def.getPointer());
3809 if (EliminationStack.empty() ||
3810 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)) {
3811 // Sync to our current scope.
3812 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
3813 if (EliminationStack.empty()) {
3814 EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);
3818 // We already did load elimination, so nothing to do here.
3819 if (isa<LoadInst>(Member))
3821 assert(!EliminationStack.empty());
3822 Instruction *Leader = cast<Instruction>(EliminationStack.back());
3824 assert(DT->dominates(Leader->getParent(), Member->getParent()));
3825 // Member is dominater by Leader, and thus dead
3826 DEBUG(dbgs() << "Marking dead store " << *Member
3827 << " that is dominated by " << *Leader << "\n");
3828 markInstructionForDeletion(Member);
3834 return AnythingReplaced;
3837 // This function provides global ranking of operations so that we can place them
3838 // in a canonical order. Note that rank alone is not necessarily enough for a
3839 // complete ordering, as constants all have the same rank. However, generally,
3840 // we will simplify an operation with all constants so that it doesn't matter
3841 // what order they appear in.
3842 unsigned int NewGVN::getRank(const Value *V) const {
3843 // Prefer constants to undef to anything else
3844 // Undef is a constant, have to check it first.
3845 // Prefer smaller constants to constantexprs
3846 if (isa<ConstantExpr>(V))
3848 if (isa<UndefValue>(V))
3850 if (isa<Constant>(V))
3852 else if (auto *A = dyn_cast<Argument>(V))
3853 return 3 + A->getArgNo();
3855 // Need to shift the instruction DFS by number of arguments + 3 to account for
3856 // the constant and argument ranking above.
3857 unsigned Result = InstrToDFSNum(V);
3859 return 4 + NumFuncArgs + Result;
3860 // Unreachable or something else, just return a really large number.
3864 // This is a function that says whether two commutative operations should
3865 // have their order swapped when canonicalizing.
3866 bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
3867 // Because we only care about a total ordering, and don't rewrite expressions
3868 // in this order, we order by rank, which will give a strict weak ordering to
3869 // everything but constants, and then we order by pointer address.
3870 return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);
3874 class NewGVNLegacyPass : public FunctionPass {
3876 static char ID; // Pass identification, replacement for typeid.
3877 NewGVNLegacyPass() : FunctionPass(ID) {
3878 initializeNewGVNLegacyPassPass(*PassRegistry::getPassRegistry());
3880 bool runOnFunction(Function &F) override;
3883 void getAnalysisUsage(AnalysisUsage &AU) const override {
3884 AU.addRequired<AssumptionCacheTracker>();
3885 AU.addRequired<DominatorTreeWrapperPass>();
3886 AU.addRequired<TargetLibraryInfoWrapperPass>();
3887 AU.addRequired<MemorySSAWrapperPass>();
3888 AU.addRequired<AAResultsWrapperPass>();
3889 AU.addPreserved<DominatorTreeWrapperPass>();
3890 AU.addPreserved<GlobalsAAWrapperPass>();
3895 bool NewGVNLegacyPass::runOnFunction(Function &F) {
3896 if (skipFunction(F))
3898 return NewGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
3899 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
3900 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
3901 &getAnalysis<AAResultsWrapperPass>().getAAResults(),
3902 &getAnalysis<MemorySSAWrapperPass>().getMSSA(),
3903 F.getParent()->getDataLayout())
3907 INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering",
3909 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
3910 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
3911 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
3912 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
3913 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
3914 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
3915 INITIALIZE_PASS_END(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false,
3918 char NewGVNLegacyPass::ID = 0;
3920 // createGVNPass - The public interface to this file.
3921 FunctionPass *llvm::createNewGVNPass() { return new NewGVNLegacyPass(); }
3923 PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
3924 // Apparently the order in which we get these results matter for
3925 // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
3926 // the same order here, just in case.
3927 auto &AC = AM.getResult<AssumptionAnalysis>(F);
3928 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
3929 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
3930 auto &AA = AM.getResult<AAManager>(F);
3931 auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
3933 NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getParent()->getDataLayout())
3936 return PreservedAnalyses::all();
3937 PreservedAnalyses PA;
3938 PA.preserve<DominatorTreeAnalysis>();
3939 PA.preserve<GlobalsAA>();