1 //===- NewGVN.cpp - Global Value Numbering Pass ---------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// This file implements the new LLVM's Global Value Numbering pass.
12 /// GVN partitions values computed by a function into congruence classes.
13 /// Values ending up in the same congruence class are guaranteed to be the same
14 /// for every execution of the program. In that respect, congruency is a
15 /// compile-time approximation of equivalence of values at runtime.
16 /// The algorithm implemented here uses a sparse formulation and it's based
17 /// on the ideas described in the paper:
18 /// "A Sparse Algorithm for Predicated Global Value Numbering" from
21 /// A brief overview of the algorithm: The algorithm is essentially the same as
22 /// the standard RPO value numbering algorithm (a good reference is the paper
23 /// "SCC based value numbering" by L. Taylor Simpson) with one major difference:
24 /// The RPO algorithm proceeds, on every iteration, to process every reachable
25 /// block and every instruction in that block. This is because the standard RPO
26 /// algorithm does not track what things have the same value number, it only
27 /// tracks what the value number of a given operation is (the mapping is
28 /// operation -> value number). Thus, when a value number of an operation
29 /// changes, it must reprocess everything to ensure all uses of a value number
30 /// get updated properly. In constrast, the sparse algorithm we use *also*
31 /// tracks what operations have a given value number (IE it also tracks the
32 /// reverse mapping from value number -> operations with that value number), so
33 /// that it only needs to reprocess the instructions that are affected when
34 /// something's value number changes. The vast majority of complexity and code
35 /// in this file is devoted to tracking what value numbers could change for what
36 /// instructions when various things happen. The rest of the algorithm is
37 /// devoted to performing symbolic evaluation, forward propagation, and
38 /// simplification of operations based on the value numbers deduced so far
40 /// In order to make the GVN mostly-complete, we use a technique derived from
41 /// "Detection of Redundant Expressions: A Complete and Polynomial-time
42 /// Algorithm in SSA" by R.R. Pai. The source of incompleteness in most SSA
43 /// based GVN algorithms is related to their inability to detect equivalence
44 /// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)).
45 /// We resolve this issue by generating the equivalent "phi of ops" form for
46 /// each op of phis we see, in a way that only takes polynomial time to resolve.
48 /// We also do not perform elimination by using any published algorithm. All
49 /// published algorithms are O(Instructions). Instead, we use a technique that
50 /// is O(number of operations with the same value number), enabling us to skip
51 /// trying to eliminate things that have unique value numbers.
53 //===----------------------------------------------------------------------===//
55 #include "llvm/Transforms/Scalar/NewGVN.h"
56 #include "llvm/ADT/ArrayRef.h"
57 #include "llvm/ADT/BitVector.h"
58 #include "llvm/ADT/DenseMap.h"
59 #include "llvm/ADT/DenseMapInfo.h"
60 #include "llvm/ADT/DenseSet.h"
61 #include "llvm/ADT/DepthFirstIterator.h"
62 #include "llvm/ADT/GraphTraits.h"
63 #include "llvm/ADT/Hashing.h"
64 #include "llvm/ADT/PointerIntPair.h"
65 #include "llvm/ADT/PostOrderIterator.h"
66 #include "llvm/ADT/SmallPtrSet.h"
67 #include "llvm/ADT/SmallVector.h"
68 #include "llvm/ADT/SparseBitVector.h"
69 #include "llvm/ADT/Statistic.h"
70 #include "llvm/ADT/iterator_range.h"
71 #include "llvm/Analysis/AliasAnalysis.h"
72 #include "llvm/Analysis/AssumptionCache.h"
73 #include "llvm/Analysis/CFGPrinter.h"
74 #include "llvm/Analysis/ConstantFolding.h"
75 #include "llvm/Analysis/GlobalsModRef.h"
76 #include "llvm/Analysis/InstructionSimplify.h"
77 #include "llvm/Analysis/MemoryBuiltins.h"
78 #include "llvm/Analysis/MemorySSA.h"
79 #include "llvm/Analysis/TargetLibraryInfo.h"
80 #include "llvm/Transforms/Utils/Local.h"
81 #include "llvm/IR/Argument.h"
82 #include "llvm/IR/BasicBlock.h"
83 #include "llvm/IR/Constant.h"
84 #include "llvm/IR/Constants.h"
85 #include "llvm/IR/Dominators.h"
86 #include "llvm/IR/Function.h"
87 #include "llvm/IR/InstrTypes.h"
88 #include "llvm/IR/Instruction.h"
89 #include "llvm/IR/Instructions.h"
90 #include "llvm/IR/IntrinsicInst.h"
91 #include "llvm/IR/Intrinsics.h"
92 #include "llvm/IR/LLVMContext.h"
93 #include "llvm/IR/Type.h"
94 #include "llvm/IR/Use.h"
95 #include "llvm/IR/User.h"
96 #include "llvm/IR/Value.h"
97 #include "llvm/Pass.h"
98 #include "llvm/Support/Allocator.h"
99 #include "llvm/Support/ArrayRecycler.h"
100 #include "llvm/Support/Casting.h"
101 #include "llvm/Support/CommandLine.h"
102 #include "llvm/Support/Debug.h"
103 #include "llvm/Support/DebugCounter.h"
104 #include "llvm/Support/ErrorHandling.h"
105 #include "llvm/Support/PointerLikeTypeTraits.h"
106 #include "llvm/Support/raw_ostream.h"
107 #include "llvm/Transforms/Scalar.h"
108 #include "llvm/Transforms/Scalar/GVNExpression.h"
109 #include "llvm/Transforms/Utils/PredicateInfo.h"
110 #include "llvm/Transforms/Utils/VNCoercion.h"
123 using namespace llvm;
124 using namespace llvm::GVNExpression;
125 using namespace llvm::VNCoercion;
127 #define DEBUG_TYPE "newgvn"
129 STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
130 STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
131 STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
132 STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
133 STATISTIC(NumGVNMaxIterations,
134 "Maximum Number of iterations it took to converge GVN");
135 STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
136 STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
137 STATISTIC(NumGVNAvoidedSortedLeaderChanges,
138 "Number of avoided sorted leader changes");
139 STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");
140 STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created");
141 STATISTIC(NumGVNPHIOfOpsEliminations,
142 "Number of things eliminated using PHI of ops");
143 DEBUG_COUNTER(VNCounter, "newgvn-vn",
144 "Controls which instructions are value numbered");
145 DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",
146 "Controls which instructions we create phi of ops for");
147 // Currently store defining access refinement is too slow due to basicaa being
148 // egregiously slow. This flag lets us keep it working while we work on this
150 static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
151 cl::init(false), cl::Hidden);
153 /// Currently, the generation "phi of ops" can result in correctness issues.
154 static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(true),
157 //===----------------------------------------------------------------------===//
159 //===----------------------------------------------------------------------===//
163 namespace GVNExpression {
165 Expression::~Expression() = default;
166 BasicExpression::~BasicExpression() = default;
167 CallExpression::~CallExpression() = default;
168 LoadExpression::~LoadExpression() = default;
169 StoreExpression::~StoreExpression() = default;
170 AggregateValueExpression::~AggregateValueExpression() = default;
171 PHIExpression::~PHIExpression() = default;
173 } // end namespace GVNExpression
174 } // end namespace llvm
178 // Tarjan's SCC finding algorithm with Nuutila's improvements
179 // SCCIterator is actually fairly complex for the simple thing we want.
180 // It also wants to hand us SCC's that are unrelated to the phi node we ask
181 // about, and have us process them there or risk redoing work.
182 // Graph traits over a filter iterator also doesn't work that well here.
183 // This SCC finder is specialized to walk use-def chains, and only follows
185 // not generic values (arguments, etc).
187 TarjanSCC() : Components(1) {}
189 void Start(const Instruction *Start) {
190 if (Root.lookup(Start) == 0)
194 const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {
195 unsigned ComponentID = ValueToComponent.lookup(V);
197 assert(ComponentID > 0 &&
198 "Asking for a component for a value we never processed");
199 return Components[ComponentID];
203 void FindSCC(const Instruction *I) {
205 // Store the DFS Number we had before it possibly gets incremented.
206 unsigned int OurDFS = DFSNum;
207 for (auto &Op : I->operands()) {
208 if (auto *InstOp = dyn_cast<Instruction>(Op)) {
209 if (Root.lookup(Op) == 0)
211 if (!InComponent.count(Op))
212 Root[I] = std::min(Root.lookup(I), Root.lookup(Op));
215 // See if we really were the root of a component, by seeing if we still have
216 // our DFSNumber. If we do, we are the root of the component, and we have
217 // completed a component. If we do not, we are not the root of a component,
218 // and belong on the component stack.
219 if (Root.lookup(I) == OurDFS) {
220 unsigned ComponentID = Components.size();
221 Components.resize(Components.size() + 1);
222 auto &Component = Components.back();
224 LLVM_DEBUG(dbgs() << "Component root is " << *I << "\n");
225 InComponent.insert(I);
226 ValueToComponent[I] = ComponentID;
227 // Pop a component off the stack and label it.
228 while (!Stack.empty() && Root.lookup(Stack.back()) >= OurDFS) {
229 auto *Member = Stack.back();
230 LLVM_DEBUG(dbgs() << "Component member is " << *Member << "\n");
231 Component.insert(Member);
232 InComponent.insert(Member);
233 ValueToComponent[Member] = ComponentID;
237 // Part of a component, push to stack
242 unsigned int DFSNum = 1;
243 SmallPtrSet<const Value *, 8> InComponent;
244 DenseMap<const Value *, unsigned int> Root;
245 SmallVector<const Value *, 8> Stack;
247 // Store the components as vector of ptr sets, because we need the topo order
248 // of SCC's, but not individual member order
249 SmallVector<SmallPtrSet<const Value *, 8>, 8> Components;
251 DenseMap<const Value *, unsigned> ValueToComponent;
254 // Congruence classes represent the set of expressions/instructions
255 // that are all the same *during some scope in the function*.
256 // That is, because of the way we perform equality propagation, and
257 // because of memory value numbering, it is not correct to assume
258 // you can willy-nilly replace any member with any other at any
259 // point in the function.
261 // For any Value in the Member set, it is valid to replace any dominated member
264 // Every congruence class has a leader, and the leader is used to symbolize
265 // instructions in a canonical way (IE every operand of an instruction that is a
266 // member of the same congruence class will always be replaced with leader
267 // during symbolization). To simplify symbolization, we keep the leader as a
268 // constant if class can be proved to be a constant value. Otherwise, the
269 // leader is the member of the value set with the smallest DFS number. Each
270 // congruence class also has a defining expression, though the expression may be
271 // null. If it exists, it can be used for forward propagation and reassociation
274 // For memory, we also track a representative MemoryAccess, and a set of memory
275 // members for MemoryPhis (which have no real instructions). Note that for
276 // memory, it seems tempting to try to split the memory members into a
277 // MemoryCongruenceClass or something. Unfortunately, this does not work
278 // easily. The value numbering of a given memory expression depends on the
279 // leader of the memory congruence class, and the leader of memory congruence
280 // class depends on the value numbering of a given memory expression. This
281 // leads to wasted propagation, and in some cases, missed optimization. For
282 // example: If we had value numbered two stores together before, but now do not,
283 // we move them to a new value congruence class. This in turn will move at one
284 // of the memorydefs to a new memory congruence class. Which in turn, affects
285 // the value numbering of the stores we just value numbered (because the memory
286 // congruence class is part of the value number). So while theoretically
287 // possible to split them up, it turns out to be *incredibly* complicated to get
288 // it to work right, because of the interdependency. While structurally
289 // slightly messier, it is algorithmically much simpler and faster to do what we
290 // do here, and track them both at once in the same class.
291 // Note: The default iterators for this class iterate over values
292 class CongruenceClass {
294 using MemberType = Value;
295 using MemberSet = SmallPtrSet<MemberType *, 4>;
296 using MemoryMemberType = MemoryPhi;
297 using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;
299 explicit CongruenceClass(unsigned ID) : ID(ID) {}
300 CongruenceClass(unsigned ID, Value *Leader, const Expression *E)
301 : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
303 unsigned getID() const { return ID; }
305 // True if this class has no members left. This is mainly used for assertion
306 // purposes, and for skipping empty classes.
307 bool isDead() const {
308 // If it's both dead from a value perspective, and dead from a memory
309 // perspective, it's really dead.
310 return empty() && memory_empty();
314 Value *getLeader() const { return RepLeader; }
315 void setLeader(Value *Leader) { RepLeader = Leader; }
316 const std::pair<Value *, unsigned int> &getNextLeader() const {
319 void resetNextLeader() { NextLeader = {nullptr, ~0}; }
320 void addPossibleNextLeader(std::pair<Value *, unsigned int> LeaderPair) {
321 if (LeaderPair.second < NextLeader.second)
322 NextLeader = LeaderPair;
325 Value *getStoredValue() const { return RepStoredValue; }
326 void setStoredValue(Value *Leader) { RepStoredValue = Leader; }
327 const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }
328 void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }
330 // Forward propagation info
331 const Expression *getDefiningExpr() const { return DefiningExpr; }
334 bool empty() const { return Members.empty(); }
335 unsigned size() const { return Members.size(); }
336 MemberSet::const_iterator begin() const { return Members.begin(); }
337 MemberSet::const_iterator end() const { return Members.end(); }
338 void insert(MemberType *M) { Members.insert(M); }
339 void erase(MemberType *M) { Members.erase(M); }
340 void swap(MemberSet &Other) { Members.swap(Other); }
343 bool memory_empty() const { return MemoryMembers.empty(); }
344 unsigned memory_size() const { return MemoryMembers.size(); }
345 MemoryMemberSet::const_iterator memory_begin() const {
346 return MemoryMembers.begin();
348 MemoryMemberSet::const_iterator memory_end() const {
349 return MemoryMembers.end();
351 iterator_range<MemoryMemberSet::const_iterator> memory() const {
352 return make_range(memory_begin(), memory_end());
355 void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); }
356 void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); }
359 unsigned getStoreCount() const { return StoreCount; }
360 void incStoreCount() { ++StoreCount; }
361 void decStoreCount() {
362 assert(StoreCount != 0 && "Store count went negative");
366 // True if this class has no memory members.
367 bool definesNoMemory() const { return StoreCount == 0 && memory_empty(); }
369 // Return true if two congruence classes are equivalent to each other. This
370 // means that every field but the ID number and the dead field are equivalent.
371 bool isEquivalentTo(const CongruenceClass *Other) const {
377 if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) !=
378 std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue,
379 Other->RepMemoryAccess))
381 if (DefiningExpr != Other->DefiningExpr)
382 if (!DefiningExpr || !Other->DefiningExpr ||
383 *DefiningExpr != *Other->DefiningExpr)
386 if (Members.size() != Other->Members.size())
389 return all_of(Members,
390 [&](const Value *V) { return Other->Members.count(V); });
396 // Representative leader.
397 Value *RepLeader = nullptr;
399 // The most dominating leader after our current leader, because the member set
400 // is not sorted and is expensive to keep sorted all the time.
401 std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
403 // If this is represented by a store, the value of the store.
404 Value *RepStoredValue = nullptr;
406 // If this class contains MemoryDefs or MemoryPhis, this is the leading memory
408 const MemoryAccess *RepMemoryAccess = nullptr;
410 // Defining Expression.
411 const Expression *DefiningExpr = nullptr;
413 // Actual members of this class.
416 // This is the set of MemoryPhis that exist in the class. MemoryDefs and
417 // MemoryUses have real instructions representing them, so we only need to
418 // track MemoryPhis here.
419 MemoryMemberSet MemoryMembers;
421 // Number of stores in this congruence class.
422 // This is used so we can detect store equivalence changes properly.
426 } // end anonymous namespace
430 struct ExactEqualsExpression {
433 explicit ExactEqualsExpression(const Expression &E) : E(E) {}
435 hash_code getComputedHash() const { return E.getComputedHash(); }
437 bool operator==(const Expression &Other) const {
438 return E.exactlyEquals(Other);
442 template <> struct DenseMapInfo<const Expression *> {
443 static const Expression *getEmptyKey() {
444 auto Val = static_cast<uintptr_t>(-1);
445 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
446 return reinterpret_cast<const Expression *>(Val);
449 static const Expression *getTombstoneKey() {
450 auto Val = static_cast<uintptr_t>(~1U);
451 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
452 return reinterpret_cast<const Expression *>(Val);
455 static unsigned getHashValue(const Expression *E) {
456 return E->getComputedHash();
459 static unsigned getHashValue(const ExactEqualsExpression &E) {
460 return E.getComputedHash();
463 static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS) {
464 if (RHS == getTombstoneKey() || RHS == getEmptyKey())
469 static bool isEqual(const Expression *LHS, const Expression *RHS) {
472 if (LHS == getTombstoneKey() || RHS == getTombstoneKey() ||
473 LHS == getEmptyKey() || RHS == getEmptyKey())
475 // Compare hashes before equality. This is *not* what the hashtable does,
476 // since it is computing it modulo the number of buckets, whereas we are
477 // using the full hash keyspace. Since the hashes are precomputed, this
478 // check is *much* faster than equality.
479 if (LHS->getComputedHash() != RHS->getComputedHash())
485 } // end namespace llvm
492 const TargetLibraryInfo *TLI;
495 MemorySSAWalker *MSSAWalker;
496 const DataLayout &DL;
497 std::unique_ptr<PredicateInfo> PredInfo;
499 // These are the only two things the create* functions should have
500 // side-effects on due to allocating memory.
501 mutable BumpPtrAllocator ExpressionAllocator;
502 mutable ArrayRecycler<Value *> ArgRecycler;
503 mutable TarjanSCC SCCFinder;
504 const SimplifyQuery SQ;
506 // Number of function arguments, used by ranking
507 unsigned int NumFuncArgs;
509 // RPOOrdering of basic blocks
510 DenseMap<const DomTreeNode *, unsigned> RPOOrdering;
512 // Congruence class info.
514 // This class is called INITIAL in the paper. It is the class everything
515 // startsout in, and represents any value. Being an optimistic analysis,
516 // anything in the TOP class has the value TOP, which is indeterminate and
517 // equivalent to everything.
518 CongruenceClass *TOPClass;
519 std::vector<CongruenceClass *> CongruenceClasses;
520 unsigned NextCongruenceNum;
523 DenseMap<Value *, CongruenceClass *> ValueToClass;
524 DenseMap<Value *, const Expression *> ValueToExpression;
526 // Value PHI handling, used to make equivalence between phi(op, op) and
528 // These mappings just store various data that would normally be part of the
530 SmallPtrSet<const Instruction *, 8> PHINodeUses;
532 DenseMap<const Value *, bool> OpSafeForPHIOfOps;
534 // Map a temporary instruction we created to a parent block.
535 DenseMap<const Value *, BasicBlock *> TempToBlock;
537 // Map between the already in-program instructions and the temporary phis we
538 // created that they are known equivalent to.
539 DenseMap<const Value *, PHINode *> RealToTemp;
541 // In order to know when we should re-process instructions that have
542 // phi-of-ops, we track the set of expressions that they needed as
543 // leaders. When we discover new leaders for those expressions, we process the
544 // associated phi-of-op instructions again in case they have changed. The
545 // other way they may change is if they had leaders, and those leaders
546 // disappear. However, at the point they have leaders, there are uses of the
547 // relevant operands in the created phi node, and so they will get reprocessed
548 // through the normal user marking we perform.
549 mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers;
550 DenseMap<const Expression *, SmallPtrSet<Instruction *, 2>>
551 ExpressionToPhiOfOps;
553 // Map from temporary operation to MemoryAccess.
554 DenseMap<const Instruction *, MemoryUseOrDef *> TempToMemory;
556 // Set of all temporary instructions we created.
557 // Note: This will include instructions that were just created during value
558 // numbering. The way to test if something is using them is to check
560 DenseSet<Instruction *> AllTempInstructions;
562 // This is the set of instructions to revisit on a reachability change. At
563 // the end of the main iteration loop it will contain at least all the phi of
564 // ops instructions that will be changed to phis, as well as regular phis.
565 // During the iteration loop, it may contain other things, such as phi of ops
566 // instructions that used edge reachability to reach a result, and so need to
567 // be revisited when the edge changes, independent of whether the phi they
568 // depended on changes.
569 DenseMap<BasicBlock *, SparseBitVector<>> RevisitOnReachabilityChange;
571 // Mapping from predicate info we used to the instructions we used it with.
572 // In order to correctly ensure propagation, we must keep track of what
573 // comparisons we used, so that when the values of the comparisons change, we
574 // propagate the information to the places we used the comparison.
575 mutable DenseMap<const Value *, SmallPtrSet<Instruction *, 2>>
578 // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for
579 // stores, we no longer can rely solely on the def-use chains of MemorySSA.
580 mutable DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>>
583 // A table storing which memorydefs/phis represent a memory state provably
584 // equivalent to another memory state.
585 // We could use the congruence class machinery, but the MemoryAccess's are
586 // abstract memory states, so they can only ever be equivalent to each other,
587 // and not to constants, etc.
588 DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass;
590 // We could, if we wanted, build MemoryPhiExpressions and
591 // MemoryVariableExpressions, etc, and value number them the same way we value
592 // number phi expressions. For the moment, this seems like overkill. They
593 // can only exist in one of three states: they can be TOP (equal to
594 // everything), Equivalent to something else, or unique. Because we do not
595 // create expressions for them, we need to simulate leader change not just
596 // when they change class, but when they change state. Note: We can do the
597 // same thing for phis, and avoid having phi expressions if we wanted, We
598 // should eventually unify in one direction or the other, so this is a little
599 // bit of an experiment in which turns out easier to maintain.
600 enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };
601 DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState;
603 enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle };
604 mutable DenseMap<const Instruction *, InstCycleState> InstCycleState;
606 // Expression to class mapping.
607 using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
608 ExpressionClassMap ExpressionToClass;
610 // We have a single expression that represents currently DeadExpressions.
611 // For dead expressions we can prove will stay dead, we mark them with
612 // DFS number zero. However, it's possible in the case of phi nodes
613 // for us to assume/prove all arguments are dead during fixpointing.
614 // We use DeadExpression for that case.
615 DeadExpression *SingletonDeadExpression = nullptr;
617 // Which values have changed as a result of leader changes.
618 SmallPtrSet<Value *, 8> LeaderChanges;
620 // Reachability info.
621 using BlockEdge = BasicBlockEdge;
622 DenseSet<BlockEdge> ReachableEdges;
623 SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
625 // This is a bitvector because, on larger functions, we may have
626 // thousands of touched instructions at once (entire blocks,
627 // instructions with hundreds of uses, etc). Even with optimization
628 // for when we mark whole blocks as touched, when this was a
629 // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
630 // the time in GVN just managing this list. The bitvector, on the
631 // other hand, efficiently supports test/set/clear of both
632 // individual and ranges, as well as "find next element" This
633 // enables us to use it as a worklist with essentially 0 cost.
634 BitVector TouchedInstructions;
636 DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
639 // Debugging for how many times each block and instruction got processed.
640 DenseMap<const Value *, unsigned> ProcessedCount;
644 // This contains a mapping from Instructions to DFS numbers.
645 // The numbering starts at 1. An instruction with DFS number zero
646 // means that the instruction is dead.
647 DenseMap<const Value *, unsigned> InstrDFS;
649 // This contains the mapping DFS numbers to instructions.
650 SmallVector<Value *, 32> DFSToInstr;
653 SmallPtrSet<Instruction *, 8> InstructionsToErase;
656 NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
657 TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
658 const DataLayout &DL)
659 : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), DL(DL),
660 PredInfo(make_unique<PredicateInfo>(F, *DT, *AC)),
661 SQ(DL, TLI, DT, AC, /*CtxI=*/nullptr, /*UseInstrInfo=*/false) {}
666 // Expression handling.
667 const Expression *createExpression(Instruction *) const;
668 const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,
669 Instruction *) const;
671 // Our canonical form for phi arguments is a pair of incoming value, incoming
673 using ValPair = std::pair<Value *, BasicBlock *>;
675 PHIExpression *createPHIExpression(ArrayRef<ValPair>, const Instruction *,
676 BasicBlock *, bool &HasBackEdge,
677 bool &OriginalOpsConstant) const;
678 const DeadExpression *createDeadExpression() const;
679 const VariableExpression *createVariableExpression(Value *) const;
680 const ConstantExpression *createConstantExpression(Constant *) const;
681 const Expression *createVariableOrConstant(Value *V) const;
682 const UnknownExpression *createUnknownExpression(Instruction *) const;
683 const StoreExpression *createStoreExpression(StoreInst *,
684 const MemoryAccess *) const;
685 LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
686 const MemoryAccess *) const;
687 const CallExpression *createCallExpression(CallInst *,
688 const MemoryAccess *) const;
689 const AggregateValueExpression *
690 createAggregateValueExpression(Instruction *) const;
691 bool setBasicExpressionInfo(Instruction *, BasicExpression *) const;
693 // Congruence class handling.
694 CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
695 auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E);
696 CongruenceClasses.emplace_back(result);
700 CongruenceClass *createMemoryClass(MemoryAccess *MA) {
701 auto *CC = createCongruenceClass(nullptr, nullptr);
702 CC->setMemoryLeader(MA);
706 CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {
707 auto *CC = getMemoryClass(MA);
708 if (CC->getMemoryLeader() != MA)
709 CC = createMemoryClass(MA);
713 CongruenceClass *createSingletonCongruenceClass(Value *Member) {
714 CongruenceClass *CClass = createCongruenceClass(Member, nullptr);
715 CClass->insert(Member);
716 ValueToClass[Member] = CClass;
720 void initializeCongruenceClasses(Function &F);
721 const Expression *makePossiblePHIOfOps(Instruction *,
722 SmallPtrSetImpl<Value *> &);
723 Value *findLeaderForInst(Instruction *ValueOp,
724 SmallPtrSetImpl<Value *> &Visited,
725 MemoryAccess *MemAccess, Instruction *OrigInst,
727 bool OpIsSafeForPHIOfOpsHelper(Value *V, const BasicBlock *PHIBlock,
728 SmallPtrSetImpl<const Value *> &Visited,
729 SmallVectorImpl<Instruction *> &Worklist);
730 bool OpIsSafeForPHIOfOps(Value *Op, const BasicBlock *PHIBlock,
731 SmallPtrSetImpl<const Value *> &);
732 void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue);
733 void removePhiOfOps(Instruction *I, PHINode *PHITemp);
735 // Value number an Instruction or MemoryPhi.
736 void valueNumberMemoryPhi(MemoryPhi *);
737 void valueNumberInstruction(Instruction *);
739 // Symbolic evaluation.
740 const Expression *checkSimplificationResults(Expression *, Instruction *,
742 const Expression *performSymbolicEvaluation(Value *,
743 SmallPtrSetImpl<Value *> &) const;
744 const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
746 MemoryAccess *) const;
747 const Expression *performSymbolicLoadEvaluation(Instruction *) const;
748 const Expression *performSymbolicStoreEvaluation(Instruction *) const;
749 const Expression *performSymbolicCallEvaluation(Instruction *) const;
750 void sortPHIOps(MutableArrayRef<ValPair> Ops) const;
751 const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>,
753 BasicBlock *PHIBlock) const;
754 const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
755 const Expression *performSymbolicCmpEvaluation(Instruction *) const;
756 const Expression *performSymbolicPredicateInfoEvaluation(Instruction *) const;
758 // Congruence finding.
759 bool someEquivalentDominates(const Instruction *, const Instruction *) const;
760 Value *lookupOperandLeader(Value *) const;
761 CongruenceClass *getClassForExpression(const Expression *E) const;
762 void performCongruenceFinding(Instruction *, const Expression *);
763 void moveValueToNewCongruenceClass(Instruction *, const Expression *,
764 CongruenceClass *, CongruenceClass *);
765 void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,
766 CongruenceClass *, CongruenceClass *);
767 Value *getNextValueLeader(CongruenceClass *) const;
768 const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;
769 bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);
770 CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;
771 const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;
772 bool isMemoryAccessTOP(const MemoryAccess *) const;
775 unsigned int getRank(const Value *) const;
776 bool shouldSwapOperands(const Value *, const Value *) const;
778 // Reachability handling.
779 void updateReachableEdge(BasicBlock *, BasicBlock *);
780 void processOutgoingEdges(Instruction *, BasicBlock *);
781 Value *findConditionEquivalence(Value *) const;
785 void convertClassToDFSOrdered(const CongruenceClass &,
786 SmallVectorImpl<ValueDFS> &,
787 DenseMap<const Value *, unsigned int> &,
788 SmallPtrSetImpl<Instruction *> &) const;
789 void convertClassToLoadsAndStores(const CongruenceClass &,
790 SmallVectorImpl<ValueDFS> &) const;
792 bool eliminateInstructions(Function &);
793 void replaceInstruction(Instruction *, Value *);
794 void markInstructionForDeletion(Instruction *);
795 void deleteInstructionsInBlock(BasicBlock *);
796 Value *findPHIOfOpsLeader(const Expression *, const Instruction *,
797 const BasicBlock *) const;
799 // New instruction creation.
800 void handleNewInstruction(Instruction *) {}
802 // Various instruction touch utilities
803 template <typename Map, typename KeyType, typename Func>
804 void for_each_found(Map &, const KeyType &, Func);
805 template <typename Map, typename KeyType>
806 void touchAndErase(Map &, const KeyType &);
807 void markUsersTouched(Value *);
808 void markMemoryUsersTouched(const MemoryAccess *);
809 void markMemoryDefTouched(const MemoryAccess *);
810 void markPredicateUsersTouched(Instruction *);
811 void markValueLeaderChangeTouched(CongruenceClass *CC);
812 void markMemoryLeaderChangeTouched(CongruenceClass *CC);
813 void markPhiOfOpsChanged(const Expression *E);
814 void addPredicateUsers(const PredicateBase *, Instruction *) const;
815 void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;
816 void addAdditionalUsers(Value *To, Value *User) const;
818 // Main loop of value numbering
819 void iterateTouchedInstructions();
822 void cleanupTables();
823 std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
824 void updateProcessedCount(const Value *V);
825 void verifyMemoryCongruency() const;
826 void verifyIterationSettled(Function &F);
827 void verifyStoreExpressions() const;
828 bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &,
829 const MemoryAccess *, const MemoryAccess *) const;
830 BasicBlock *getBlockForValue(Value *V) const;
831 void deleteExpression(const Expression *E) const;
832 MemoryUseOrDef *getMemoryAccess(const Instruction *) const;
833 MemoryAccess *getDefiningAccess(const MemoryAccess *) const;
834 MemoryPhi *getMemoryAccess(const BasicBlock *) const;
835 template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
837 unsigned InstrToDFSNum(const Value *V) const {
838 assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");
839 return InstrDFS.lookup(V);
842 unsigned InstrToDFSNum(const MemoryAccess *MA) const {
843 return MemoryToDFSNum(MA);
846 Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }
848 // Given a MemoryAccess, return the relevant instruction DFS number. Note:
849 // This deliberately takes a value so it can be used with Use's, which will
850 // auto-convert to Value's but not to MemoryAccess's.
851 unsigned MemoryToDFSNum(const Value *MA) const {
852 assert(isa<MemoryAccess>(MA) &&
853 "This should not be used with instructions");
854 return isa<MemoryUseOrDef>(MA)
855 ? InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst())
856 : InstrDFS.lookup(MA);
859 bool isCycleFree(const Instruction *) const;
860 bool isBackedge(BasicBlock *From, BasicBlock *To) const;
862 // Debug counter info. When verifying, we have to reset the value numbering
863 // debug counter to the same state it started in to get the same results.
864 int64_t StartingVNCounter;
867 } // end anonymous namespace
869 template <typename T>
870 static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
871 if (!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS))
873 return LHS.MemoryExpression::equals(RHS);
876 bool LoadExpression::equals(const Expression &Other) const {
877 return equalsLoadStoreHelper(*this, Other);
880 bool StoreExpression::equals(const Expression &Other) const {
881 if (!equalsLoadStoreHelper(*this, Other))
883 // Make sure that store vs store includes the value operand.
884 if (const auto *S = dyn_cast<StoreExpression>(&Other))
885 if (getStoredValue() != S->getStoredValue())
890 // Determine if the edge From->To is a backedge
891 bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const {
893 RPOOrdering.lookup(DT->getNode(From)) >=
894 RPOOrdering.lookup(DT->getNode(To));
898 static std::string getBlockName(const BasicBlock *B) {
899 return DOTGraphTraits<const Function *>::getSimpleNodeLabel(B, nullptr);
903 // Get a MemoryAccess for an instruction, fake or real.
904 MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const {
905 auto *Result = MSSA->getMemoryAccess(I);
906 return Result ? Result : TempToMemory.lookup(I);
909 // Get a MemoryPhi for a basic block. These are all real.
910 MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const {
911 return MSSA->getMemoryAccess(BB);
914 // Get the basic block from an instruction/memory value.
915 BasicBlock *NewGVN::getBlockForValue(Value *V) const {
916 if (auto *I = dyn_cast<Instruction>(V)) {
917 auto *Parent = I->getParent();
920 Parent = TempToBlock.lookup(V);
921 assert(Parent && "Every fake instruction should have a block");
925 auto *MP = dyn_cast<MemoryPhi>(V);
926 assert(MP && "Should have been an instruction or a MemoryPhi");
927 return MP->getBlock();
930 // Delete a definitely dead expression, so it can be reused by the expression
931 // allocator. Some of these are not in creation functions, so we have to accept
933 void NewGVN::deleteExpression(const Expression *E) const {
934 assert(isa<BasicExpression>(E));
935 auto *BE = cast<BasicExpression>(E);
936 const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler);
937 ExpressionAllocator.Deallocate(E);
940 // If V is a predicateinfo copy, get the thing it is a copy of.
941 static Value *getCopyOf(const Value *V) {
942 if (auto *II = dyn_cast<IntrinsicInst>(V))
943 if (II->getIntrinsicID() == Intrinsic::ssa_copy)
944 return II->getOperand(0);
948 // Return true if V is really PN, even accounting for predicateinfo copies.
949 static bool isCopyOfPHI(const Value *V, const PHINode *PN) {
950 return V == PN || getCopyOf(V) == PN;
953 static bool isCopyOfAPHI(const Value *V) {
954 auto *CO = getCopyOf(V);
955 return CO && isa<PHINode>(CO);
958 // Sort PHI Operands into a canonical order. What we use here is an RPO
959 // order. The BlockInstRange numbers are generated in an RPO walk of the basic
961 void NewGVN::sortPHIOps(MutableArrayRef<ValPair> Ops) const {
962 llvm::sort(Ops, [&](const ValPair &P1, const ValPair &P2) {
963 return BlockInstRange.lookup(P1.second).first <
964 BlockInstRange.lookup(P2.second).first;
968 // Return true if V is a value that will always be available (IE can
969 // be placed anywhere) in the function. We don't do globals here
970 // because they are often worse to put in place.
971 static bool alwaysAvailable(Value *V) {
972 return isa<Constant>(V) || isa<Argument>(V);
975 // Create a PHIExpression from an array of {incoming edge, value} pairs. I is
976 // the original instruction we are creating a PHIExpression for (but may not be
977 // a phi node). We require, as an invariant, that all the PHIOperands in the
978 // same block are sorted the same way. sortPHIOps will sort them into a
980 PHIExpression *NewGVN::createPHIExpression(ArrayRef<ValPair> PHIOperands,
981 const Instruction *I,
982 BasicBlock *PHIBlock,
984 bool &OriginalOpsConstant) const {
985 unsigned NumOps = PHIOperands.size();
986 auto *E = new (ExpressionAllocator) PHIExpression(NumOps, PHIBlock);
988 E->allocateOperands(ArgRecycler, ExpressionAllocator);
989 E->setType(PHIOperands.begin()->first->getType());
990 E->setOpcode(Instruction::PHI);
992 // Filter out unreachable phi operands.
993 auto Filtered = make_filter_range(PHIOperands, [&](const ValPair &P) {
995 if (auto *PHIOp = dyn_cast<PHINode>(I))
996 if (isCopyOfPHI(P.first, PHIOp))
998 if (!ReachableEdges.count({BB, PHIBlock}))
1000 // Things in TOPClass are equivalent to everything.
1001 if (ValueToClass.lookup(P.first) == TOPClass)
1003 OriginalOpsConstant = OriginalOpsConstant && isa<Constant>(P.first);
1004 HasBackedge = HasBackedge || isBackedge(BB, PHIBlock);
1005 return lookupOperandLeader(P.first) != I;
1007 std::transform(Filtered.begin(), Filtered.end(), op_inserter(E),
1008 [&](const ValPair &P) -> Value * {
1009 return lookupOperandLeader(P.first);
1014 // Set basic expression info (Arguments, type, opcode) for Expression
1015 // E from Instruction I in block B.
1016 bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const {
1017 bool AllConstant = true;
1018 if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
1019 E->setType(GEP->getSourceElementType());
1021 E->setType(I->getType());
1022 E->setOpcode(I->getOpcode());
1023 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1025 // Transform the operand array into an operand leader array, and keep track of
1026 // whether all members are constant.
1027 std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {
1028 auto Operand = lookupOperandLeader(O);
1029 AllConstant = AllConstant && isa<Constant>(Operand);
1036 const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
1037 Value *Arg1, Value *Arg2,
1038 Instruction *I) const {
1039 auto *E = new (ExpressionAllocator) BasicExpression(2);
1042 E->setOpcode(Opcode);
1043 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1044 if (Instruction::isCommutative(Opcode)) {
1045 // Ensure that commutative instructions that only differ by a permutation
1046 // of their operands get the same value number by sorting the operand value
1047 // numbers. Since all commutative instructions have two operands it is more
1048 // efficient to sort by hand rather than using, say, std::sort.
1049 if (shouldSwapOperands(Arg1, Arg2))
1050 std::swap(Arg1, Arg2);
1052 E->op_push_back(lookupOperandLeader(Arg1));
1053 E->op_push_back(lookupOperandLeader(Arg2));
1055 Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), SQ);
1056 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1061 // Take a Value returned by simplification of Expression E/Instruction
1062 // I, and see if it resulted in a simpler expression. If so, return
1064 const Expression *NewGVN::checkSimplificationResults(Expression *E,
1069 if (auto *C = dyn_cast<Constant>(V)) {
1071 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1072 << " constant " << *C << "\n");
1073 NumGVNOpsSimplified++;
1074 assert(isa<BasicExpression>(E) &&
1075 "We should always have had a basic expression here");
1076 deleteExpression(E);
1077 return createConstantExpression(C);
1078 } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
1080 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1081 << " variable " << *V << "\n");
1082 deleteExpression(E);
1083 return createVariableExpression(V);
1086 CongruenceClass *CC = ValueToClass.lookup(V);
1088 if (CC->getLeader() && CC->getLeader() != I) {
1089 // If we simplified to something else, we need to communicate
1090 // that we're users of the value we simplified to.
1092 // Don't add temporary instructions to the user lists.
1093 if (!AllTempInstructions.count(I))
1094 addAdditionalUsers(V, I);
1096 return createVariableOrConstant(CC->getLeader());
1098 if (CC->getDefiningExpr()) {
1099 // If we simplified to something else, we need to communicate
1100 // that we're users of the value we simplified to.
1102 // Don't add temporary instructions to the user lists.
1103 if (!AllTempInstructions.count(I))
1104 addAdditionalUsers(V, I);
1108 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1109 << " expression " << *CC->getDefiningExpr() << "\n");
1110 NumGVNOpsSimplified++;
1111 deleteExpression(E);
1112 return CC->getDefiningExpr();
1119 // Create a value expression from the instruction I, replacing operands with
1122 const Expression *NewGVN::createExpression(Instruction *I) const {
1123 auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
1125 bool AllConstant = setBasicExpressionInfo(I, E);
1127 if (I->isCommutative()) {
1128 // Ensure that commutative instructions that only differ by a permutation
1129 // of their operands get the same value number by sorting the operand value
1130 // numbers. Since all commutative instructions have two operands it is more
1131 // efficient to sort by hand rather than using, say, std::sort.
1132 assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
1133 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))
1134 E->swapOperands(0, 1);
1136 // Perform simplification.
1137 if (auto *CI = dyn_cast<CmpInst>(I)) {
1138 // Sort the operand value numbers so x<y and y>x get the same value
1140 CmpInst::Predicate Predicate = CI->getPredicate();
1141 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) {
1142 E->swapOperands(0, 1);
1143 Predicate = CmpInst::getSwappedPredicate(Predicate);
1145 E->setOpcode((CI->getOpcode() << 8) | Predicate);
1146 // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands
1147 assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
1148 "Wrong types on cmp instruction");
1149 assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
1150 E->getOperand(1)->getType() == I->getOperand(1)->getType()));
1152 SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), SQ);
1153 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1155 } else if (isa<SelectInst>(I)) {
1156 if (isa<Constant>(E->getOperand(0)) ||
1157 E->getOperand(1) == E->getOperand(2)) {
1158 assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
1159 E->getOperand(2)->getType() == I->getOperand(2)->getType());
1160 Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
1161 E->getOperand(2), SQ);
1162 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1165 } else if (I->isBinaryOp()) {
1167 SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), SQ);
1168 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1170 } else if (auto *BI = dyn_cast<BitCastInst>(I)) {
1172 SimplifyCastInst(BI->getOpcode(), BI->getOperand(0), BI->getType(), SQ);
1173 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1175 } else if (isa<GetElementPtrInst>(I)) {
1176 Value *V = SimplifyGEPInst(
1177 E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ);
1178 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1180 } else if (AllConstant) {
1181 // We don't bother trying to simplify unless all of the operands
1183 // TODO: There are a lot of Simplify*'s we could call here, if we
1184 // wanted to. The original motivating case for this code was a
1185 // zext i1 false to i8, which we don't have an interface to
1186 // simplify (IE there is no SimplifyZExt).
1188 SmallVector<Constant *, 8> C;
1189 for (Value *Arg : E->operands())
1190 C.emplace_back(cast<Constant>(Arg));
1192 if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))
1193 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1199 const AggregateValueExpression *
1200 NewGVN::createAggregateValueExpression(Instruction *I) const {
1201 if (auto *II = dyn_cast<InsertValueInst>(I)) {
1202 auto *E = new (ExpressionAllocator)
1203 AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
1204 setBasicExpressionInfo(I, E);
1205 E->allocateIntOperands(ExpressionAllocator);
1206 std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E));
1208 } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1209 auto *E = new (ExpressionAllocator)
1210 AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
1211 setBasicExpressionInfo(EI, E);
1212 E->allocateIntOperands(ExpressionAllocator);
1213 std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E));
1216 llvm_unreachable("Unhandled type of aggregate value operation");
1219 const DeadExpression *NewGVN::createDeadExpression() const {
1220 // DeadExpression has no arguments and all DeadExpression's are the same,
1221 // so we only need one of them.
1222 return SingletonDeadExpression;
1225 const VariableExpression *NewGVN::createVariableExpression(Value *V) const {
1226 auto *E = new (ExpressionAllocator) VariableExpression(V);
1227 E->setOpcode(V->getValueID());
1231 const Expression *NewGVN::createVariableOrConstant(Value *V) const {
1232 if (auto *C = dyn_cast<Constant>(V))
1233 return createConstantExpression(C);
1234 return createVariableExpression(V);
1237 const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const {
1238 auto *E = new (ExpressionAllocator) ConstantExpression(C);
1239 E->setOpcode(C->getValueID());
1243 const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const {
1244 auto *E = new (ExpressionAllocator) UnknownExpression(I);
1245 E->setOpcode(I->getOpcode());
1249 const CallExpression *
1250 NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const {
1251 // FIXME: Add operand bundles for calls.
1253 new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);
1254 setBasicExpressionInfo(CI, E);
1258 // Return true if some equivalent of instruction Inst dominates instruction U.
1259 bool NewGVN::someEquivalentDominates(const Instruction *Inst,
1260 const Instruction *U) const {
1261 auto *CC = ValueToClass.lookup(Inst);
1262 // This must be an instruction because we are only called from phi nodes
1263 // in the case that the value it needs to check against is an instruction.
1265 // The most likely candidates for dominance are the leader and the next leader.
1266 // The leader or nextleader will dominate in all cases where there is an
1267 // equivalent that is higher up in the dom tree.
1268 // We can't *only* check them, however, because the
1269 // dominator tree could have an infinite number of non-dominating siblings
1270 // with instructions that are in the right congruence class.
1275 // Instruction U could be in H, with equivalents in every other sibling.
1276 // Depending on the rpo order picked, the leader could be the equivalent in
1277 // any of these siblings.
1280 if (alwaysAvailable(CC->getLeader()))
1282 if (DT->dominates(cast<Instruction>(CC->getLeader()), U))
1284 if (CC->getNextLeader().first &&
1285 DT->dominates(cast<Instruction>(CC->getNextLeader().first), U))
1287 return llvm::any_of(*CC, [&](const Value *Member) {
1288 return Member != CC->getLeader() &&
1289 DT->dominates(cast<Instruction>(Member), U);
1293 // See if we have a congruence class and leader for this operand, and if so,
1294 // return it. Otherwise, return the operand itself.
1295 Value *NewGVN::lookupOperandLeader(Value *V) const {
1296 CongruenceClass *CC = ValueToClass.lookup(V);
1298 // Everything in TOP is represented by undef, as it can be any value.
1299 // We do have to make sure we get the type right though, so we can't set the
1300 // RepLeader to undef.
1302 return UndefValue::get(V->getType());
1303 return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
1309 const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {
1310 auto *CC = getMemoryClass(MA);
1311 assert(CC->getMemoryLeader() &&
1312 "Every MemoryAccess should be mapped to a congruence class with a "
1313 "representative memory access");
1314 return CC->getMemoryLeader();
1317 // Return true if the MemoryAccess is really equivalent to everything. This is
1318 // equivalent to the lattice value "TOP" in most lattices. This is the initial
1319 // state of all MemoryAccesses.
1320 bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const {
1321 return getMemoryClass(MA) == TOPClass;
1324 LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
1326 const MemoryAccess *MA) const {
1328 new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));
1329 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1330 E->setType(LoadType);
1332 // Give store and loads same opcode so they value number together.
1334 E->op_push_back(PointerOp);
1336 E->setAlignment(LI->getAlignment());
1338 // TODO: Value number heap versions. We may be able to discover
1339 // things alias analysis can't on it's own (IE that a store and a
1340 // load have the same value, and thus, it isn't clobbering the load).
1344 const StoreExpression *
1345 NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const {
1346 auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand());
1347 auto *E = new (ExpressionAllocator)
1348 StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);
1349 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1350 E->setType(SI->getValueOperand()->getType());
1352 // Give store and loads same opcode so they value number together.
1354 E->op_push_back(lookupOperandLeader(SI->getPointerOperand()));
1356 // TODO: Value number heap versions. We may be able to discover
1357 // things alias analysis can't on it's own (IE that a store and a
1358 // load have the same value, and thus, it isn't clobbering the load).
1362 const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {
1363 // Unlike loads, we never try to eliminate stores, so we do not check if they
1364 // are simple and avoid value numbering them.
1365 auto *SI = cast<StoreInst>(I);
1366 auto *StoreAccess = getMemoryAccess(SI);
1367 // Get the expression, if any, for the RHS of the MemoryDef.
1368 const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();
1369 if (EnableStoreRefinement)
1370 StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess);
1371 // If we bypassed the use-def chains, make sure we add a use.
1372 StoreRHS = lookupMemoryLeader(StoreRHS);
1373 if (StoreRHS != StoreAccess->getDefiningAccess())
1374 addMemoryUsers(StoreRHS, StoreAccess);
1375 // If we are defined by ourselves, use the live on entry def.
1376 if (StoreRHS == StoreAccess)
1377 StoreRHS = MSSA->getLiveOnEntryDef();
1379 if (SI->isSimple()) {
1380 // See if we are defined by a previous store expression, it already has a
1381 // value, and it's the same value as our current store. FIXME: Right now, we
1382 // only do this for simple stores, we should expand to cover memcpys, etc.
1383 const auto *LastStore = createStoreExpression(SI, StoreRHS);
1384 const auto *LastCC = ExpressionToClass.lookup(LastStore);
1385 // We really want to check whether the expression we matched was a store. No
1386 // easy way to do that. However, we can check that the class we found has a
1387 // store, which, assuming the value numbering state is not corrupt, is
1388 // sufficient, because we must also be equivalent to that store's expression
1389 // for it to be in the same class as the load.
1390 if (LastCC && LastCC->getStoredValue() == LastStore->getStoredValue())
1392 // Also check if our value operand is defined by a load of the same memory
1393 // location, and the memory state is the same as it was then (otherwise, it
1394 // could have been overwritten later. See test32 in
1395 // transforms/DeadStoreElimination/simple.ll).
1396 if (auto *LI = dyn_cast<LoadInst>(LastStore->getStoredValue()))
1397 if ((lookupOperandLeader(LI->getPointerOperand()) ==
1398 LastStore->getOperand(0)) &&
1399 (lookupMemoryLeader(getMemoryAccess(LI)->getDefiningAccess()) ==
1402 deleteExpression(LastStore);
1405 // If the store is not equivalent to anything, value number it as a store that
1406 // produces a unique memory state (instead of using it's MemoryUse, we use
1408 return createStoreExpression(SI, StoreAccess);
1411 // See if we can extract the value of a loaded pointer from a load, a store, or
1412 // a memory instruction.
1414 NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
1415 LoadInst *LI, Instruction *DepInst,
1416 MemoryAccess *DefiningAccess) const {
1417 assert((!LI || LI->isSimple()) && "Not a simple load");
1418 if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {
1419 // Can't forward from non-atomic to atomic without violating memory model.
1420 // Also don't need to coerce if they are the same type, we will just
1422 if (LI->isAtomic() > DepSI->isAtomic() ||
1423 LoadType == DepSI->getValueOperand()->getType())
1425 int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL);
1427 if (auto *C = dyn_cast<Constant>(
1428 lookupOperandLeader(DepSI->getValueOperand()))) {
1429 LLVM_DEBUG(dbgs() << "Coercing load from store " << *DepSI
1430 << " to constant " << *C << "\n");
1431 return createConstantExpression(
1432 getConstantStoreValueForLoad(C, Offset, LoadType, DL));
1435 } else if (auto *DepLI = dyn_cast<LoadInst>(DepInst)) {
1436 // Can't forward from non-atomic to atomic without violating memory model.
1437 if (LI->isAtomic() > DepLI->isAtomic())
1439 int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);
1441 // We can coerce a constant load into a load.
1442 if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))
1443 if (auto *PossibleConstant =
1444 getConstantLoadValueForLoad(C, Offset, LoadType, DL)) {
1445 LLVM_DEBUG(dbgs() << "Coercing load from load " << *LI
1446 << " to constant " << *PossibleConstant << "\n");
1447 return createConstantExpression(PossibleConstant);
1450 } else if (auto *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
1451 int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);
1453 if (auto *PossibleConstant =
1454 getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) {
1455 LLVM_DEBUG(dbgs() << "Coercing load from meminst " << *DepMI
1456 << " to constant " << *PossibleConstant << "\n");
1457 return createConstantExpression(PossibleConstant);
1462 // All of the below are only true if the loaded pointer is produced
1463 // by the dependent instruction.
1464 if (LoadPtr != lookupOperandLeader(DepInst) &&
1465 !AA->isMustAlias(LoadPtr, DepInst))
1467 // If this load really doesn't depend on anything, then we must be loading an
1468 // undef value. This can happen when loading for a fresh allocation with no
1469 // intervening stores, for example. Note that this is only true in the case
1470 // that the result of the allocation is pointer equal to the load ptr.
1471 if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
1472 return createConstantExpression(UndefValue::get(LoadType));
1474 // If this load occurs either right after a lifetime begin,
1475 // then the loaded value is undefined.
1476 else if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {
1477 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1478 return createConstantExpression(UndefValue::get(LoadType));
1480 // If this load follows a calloc (which zero initializes memory),
1481 // then the loaded value is zero
1482 else if (isCallocLikeFn(DepInst, TLI)) {
1483 return createConstantExpression(Constant::getNullValue(LoadType));
1489 const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
1490 auto *LI = cast<LoadInst>(I);
1492 // We can eliminate in favor of non-simple loads, but we won't be able to
1493 // eliminate the loads themselves.
1494 if (!LI->isSimple())
1497 Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());
1498 // Load of undef is undef.
1499 if (isa<UndefValue>(LoadAddressLeader))
1500 return createConstantExpression(UndefValue::get(LI->getType()));
1501 MemoryAccess *OriginalAccess = getMemoryAccess(I);
1502 MemoryAccess *DefiningAccess =
1503 MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
1505 if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
1506 if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
1507 Instruction *DefiningInst = MD->getMemoryInst();
1508 // If the defining instruction is not reachable, replace with undef.
1509 if (!ReachableBlocks.count(DefiningInst->getParent()))
1510 return createConstantExpression(UndefValue::get(LI->getType()));
1511 // This will handle stores and memory insts. We only do if it the
1512 // defining access has a different type, or it is a pointer produced by
1513 // certain memory operations that cause the memory to have a fixed value
1514 // (IE things like calloc).
1515 if (const auto *CoercionResult =
1516 performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,
1517 DefiningInst, DefiningAccess))
1518 return CoercionResult;
1522 const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI,
1524 // If our MemoryLeader is not our defining access, add a use to the
1525 // MemoryLeader, so that we get reprocessed when it changes.
1526 if (LE->getMemoryLeader() != DefiningAccess)
1527 addMemoryUsers(LE->getMemoryLeader(), OriginalAccess);
1532 NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
1533 auto *PI = PredInfo->getPredicateInfoFor(I);
1537 LLVM_DEBUG(dbgs() << "Found predicate info from instruction !\n");
1539 auto *PWC = dyn_cast<PredicateWithCondition>(PI);
1543 auto *CopyOf = I->getOperand(0);
1544 auto *Cond = PWC->Condition;
1546 // If this a copy of the condition, it must be either true or false depending
1547 // on the predicate info type and edge.
1548 if (CopyOf == Cond) {
1549 // We should not need to add predicate users because the predicate info is
1550 // already a use of this operand.
1551 if (isa<PredicateAssume>(PI))
1552 return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1553 if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1554 if (PBranch->TrueEdge)
1555 return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1556 return createConstantExpression(ConstantInt::getFalse(Cond->getType()));
1558 if (auto *PSwitch = dyn_cast<PredicateSwitch>(PI))
1559 return createConstantExpression(cast<Constant>(PSwitch->CaseValue));
1562 // Not a copy of the condition, so see what the predicates tell us about this
1563 // value. First, though, we check to make sure the value is actually a copy
1564 // of one of the condition operands. It's possible, in certain cases, for it
1565 // to be a copy of a predicateinfo copy. In particular, if two branch
1566 // operations use the same condition, and one branch dominates the other, we
1567 // will end up with a copy of a copy. This is currently a small deficiency in
1568 // predicateinfo. What will end up happening here is that we will value
1569 // number both copies the same anyway.
1571 // Everything below relies on the condition being a comparison.
1572 auto *Cmp = dyn_cast<CmpInst>(Cond);
1576 if (CopyOf != Cmp->getOperand(0) && CopyOf != Cmp->getOperand(1)) {
1577 LLVM_DEBUG(dbgs() << "Copy is not of any condition operands!\n");
1580 Value *FirstOp = lookupOperandLeader(Cmp->getOperand(0));
1581 Value *SecondOp = lookupOperandLeader(Cmp->getOperand(1));
1582 bool SwappedOps = false;
1584 if (shouldSwapOperands(FirstOp, SecondOp)) {
1585 std::swap(FirstOp, SecondOp);
1588 CmpInst::Predicate Predicate =
1589 SwappedOps ? Cmp->getSwappedPredicate() : Cmp->getPredicate();
1591 if (isa<PredicateAssume>(PI)) {
1592 // If we assume the operands are equal, then they are equal.
1593 if (Predicate == CmpInst::ICMP_EQ) {
1594 addPredicateUsers(PI, I);
1595 addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0),
1597 return createVariableOrConstant(FirstOp);
1600 if (const auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1601 // If we are *not* a copy of the comparison, we may equal to the other
1602 // operand when the predicate implies something about equality of
1603 // operations. In particular, if the comparison is true/false when the
1604 // operands are equal, and we are on the right edge, we know this operation
1605 // is equal to something.
1606 if ((PBranch->TrueEdge && Predicate == CmpInst::ICMP_EQ) ||
1607 (!PBranch->TrueEdge && Predicate == CmpInst::ICMP_NE)) {
1608 addPredicateUsers(PI, I);
1609 addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0),
1611 return createVariableOrConstant(FirstOp);
1613 // Handle the special case of floating point.
1614 if (((PBranch->TrueEdge && Predicate == CmpInst::FCMP_OEQ) ||
1615 (!PBranch->TrueEdge && Predicate == CmpInst::FCMP_UNE)) &&
1616 isa<ConstantFP>(FirstOp) && !cast<ConstantFP>(FirstOp)->isZero()) {
1617 addPredicateUsers(PI, I);
1618 addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0),
1620 return createConstantExpression(cast<Constant>(FirstOp));
1626 // Evaluate read only and pure calls, and create an expression result.
1627 const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
1628 auto *CI = cast<CallInst>(I);
1629 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1630 // Intrinsics with the returned attribute are copies of arguments.
1631 if (auto *ReturnedValue = II->getReturnedArgOperand()) {
1632 if (II->getIntrinsicID() == Intrinsic::ssa_copy)
1633 if (const auto *Result = performSymbolicPredicateInfoEvaluation(I))
1635 return createVariableOrConstant(ReturnedValue);
1638 if (AA->doesNotAccessMemory(CI)) {
1639 return createCallExpression(CI, TOPClass->getMemoryLeader());
1640 } else if (AA->onlyReadsMemory(CI)) {
1641 MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI);
1642 return createCallExpression(CI, DefiningAccess);
1647 // Retrieve the memory class for a given MemoryAccess.
1648 CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {
1649 auto *Result = MemoryAccessToClass.lookup(MA);
1650 assert(Result && "Should have found memory class");
1654 // Update the MemoryAccess equivalence table to say that From is equal to To,
1655 // and return true if this is different from what already existed in the table.
1656 bool NewGVN::setMemoryClass(const MemoryAccess *From,
1657 CongruenceClass *NewClass) {
1659 "Every MemoryAccess should be getting mapped to a non-null class");
1660 LLVM_DEBUG(dbgs() << "Setting " << *From);
1661 LLVM_DEBUG(dbgs() << " equivalent to congruence class ");
1662 LLVM_DEBUG(dbgs() << NewClass->getID()
1663 << " with current MemoryAccess leader ");
1664 LLVM_DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n");
1666 auto LookupResult = MemoryAccessToClass.find(From);
1667 bool Changed = false;
1668 // If it's already in the table, see if the value changed.
1669 if (LookupResult != MemoryAccessToClass.end()) {
1670 auto *OldClass = LookupResult->second;
1671 if (OldClass != NewClass) {
1672 // If this is a phi, we have to handle memory member updates.
1673 if (auto *MP = dyn_cast<MemoryPhi>(From)) {
1674 OldClass->memory_erase(MP);
1675 NewClass->memory_insert(MP);
1676 // This may have killed the class if it had no non-memory members
1677 if (OldClass->getMemoryLeader() == From) {
1678 if (OldClass->definesNoMemory()) {
1679 OldClass->setMemoryLeader(nullptr);
1681 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1682 LLVM_DEBUG(dbgs() << "Memory class leader change for class "
1683 << OldClass->getID() << " to "
1684 << *OldClass->getMemoryLeader()
1685 << " due to removal of a memory member " << *From
1687 markMemoryLeaderChangeTouched(OldClass);
1691 // It wasn't equivalent before, and now it is.
1692 LookupResult->second = NewClass;
1700 // Determine if a instruction is cycle-free. That means the values in the
1701 // instruction don't depend on any expressions that can change value as a result
1702 // of the instruction. For example, a non-cycle free instruction would be v =
1704 bool NewGVN::isCycleFree(const Instruction *I) const {
1705 // In order to compute cycle-freeness, we do SCC finding on the instruction,
1706 // and see what kind of SCC it ends up in. If it is a singleton, it is
1707 // cycle-free. If it is not in a singleton, it is only cycle free if the
1708 // other members are all phi nodes (as they do not compute anything, they are
1710 auto ICS = InstCycleState.lookup(I);
1711 if (ICS == ICS_Unknown) {
1713 auto &SCC = SCCFinder.getComponentFor(I);
1714 // It's cycle free if it's size 1 or the SCC is *only* phi nodes.
1715 if (SCC.size() == 1)
1716 InstCycleState.insert({I, ICS_CycleFree});
1718 bool AllPhis = llvm::all_of(SCC, [](const Value *V) {
1719 return isa<PHINode>(V) || isCopyOfAPHI(V);
1721 ICS = AllPhis ? ICS_CycleFree : ICS_Cycle;
1722 for (auto *Member : SCC)
1723 if (auto *MemberPhi = dyn_cast<PHINode>(Member))
1724 InstCycleState.insert({MemberPhi, ICS});
1727 if (ICS == ICS_Cycle)
1732 // Evaluate PHI nodes symbolically and create an expression result.
1734 NewGVN::performSymbolicPHIEvaluation(ArrayRef<ValPair> PHIOps,
1736 BasicBlock *PHIBlock) const {
1737 // True if one of the incoming phi edges is a backedge.
1738 bool HasBackedge = false;
1739 // All constant tracks the state of whether all the *original* phi operands
1740 // This is really shorthand for "this phi cannot cycle due to forward
1741 // change in value of the phi is guaranteed not to later change the value of
1742 // the phi. IE it can't be v = phi(undef, v+1)
1743 bool OriginalOpsConstant = true;
1744 auto *E = cast<PHIExpression>(createPHIExpression(
1745 PHIOps, I, PHIBlock, HasBackedge, OriginalOpsConstant));
1746 // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
1747 // See if all arguments are the same.
1748 // We track if any were undef because they need special handling.
1749 bool HasUndef = false;
1750 auto Filtered = make_filter_range(E->operands(), [&](Value *Arg) {
1751 if (isa<UndefValue>(Arg)) {
1757 // If we are left with no operands, it's dead.
1758 if (empty(Filtered)) {
1759 // If it has undef at this point, it means there are no-non-undef arguments,
1760 // and thus, the value of the phi node must be undef.
1763 dbgs() << "PHI Node " << *I
1764 << " has no non-undef arguments, valuing it as undef\n");
1765 return createConstantExpression(UndefValue::get(I->getType()));
1768 LLVM_DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n");
1769 deleteExpression(E);
1770 return createDeadExpression();
1772 Value *AllSameValue = *(Filtered.begin());
1774 // Can't use std::equal here, sadly, because filter.begin moves.
1775 if (llvm::all_of(Filtered, [&](Value *Arg) { return Arg == AllSameValue; })) {
1776 // In LLVM's non-standard representation of phi nodes, it's possible to have
1777 // phi nodes with cycles (IE dependent on other phis that are .... dependent
1778 // on the original phi node), especially in weird CFG's where some arguments
1779 // are unreachable, or uninitialized along certain paths. This can cause
1780 // infinite loops during evaluation. We work around this by not trying to
1781 // really evaluate them independently, but instead using a variable
1782 // expression to say if one is equivalent to the other.
1783 // We also special case undef, so that if we have an undef, we can't use the
1784 // common value unless it dominates the phi block.
1786 // If we have undef and at least one other value, this is really a
1787 // multivalued phi, and we need to know if it's cycle free in order to
1788 // evaluate whether we can ignore the undef. The other parts of this are
1789 // just shortcuts. If there is no backedge, or all operands are
1790 // constants, it also must be cycle free.
1791 if (HasBackedge && !OriginalOpsConstant &&
1792 !isa<UndefValue>(AllSameValue) && !isCycleFree(I))
1795 // Only have to check for instructions
1796 if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))
1797 if (!someEquivalentDominates(AllSameInst, I))
1800 // Can't simplify to something that comes later in the iteration.
1801 // Otherwise, when and if it changes congruence class, we will never catch
1802 // up. We will always be a class behind it.
1803 if (isa<Instruction>(AllSameValue) &&
1804 InstrToDFSNum(AllSameValue) > InstrToDFSNum(I))
1806 NumGVNPhisAllSame++;
1807 LLVM_DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
1809 deleteExpression(E);
1810 return createVariableOrConstant(AllSameValue);
1816 NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {
1817 if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1818 auto *II = dyn_cast<IntrinsicInst>(EI->getAggregateOperand());
1819 if (II && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) {
1820 unsigned Opcode = 0;
1821 // EI might be an extract from one of our recognised intrinsics. If it
1822 // is we'll synthesize a semantically equivalent expression instead on
1823 // an extract value expression.
1824 switch (II->getIntrinsicID()) {
1825 case Intrinsic::sadd_with_overflow:
1826 case Intrinsic::uadd_with_overflow:
1827 Opcode = Instruction::Add;
1829 case Intrinsic::ssub_with_overflow:
1830 case Intrinsic::usub_with_overflow:
1831 Opcode = Instruction::Sub;
1833 case Intrinsic::smul_with_overflow:
1834 case Intrinsic::umul_with_overflow:
1835 Opcode = Instruction::Mul;
1842 // Intrinsic recognized. Grab its args to finish building the
1844 assert(II->getNumArgOperands() == 2 &&
1845 "Expect two args for recognised intrinsics.");
1846 return createBinaryExpression(Opcode, EI->getType(),
1847 II->getArgOperand(0),
1848 II->getArgOperand(1), I);
1853 return createAggregateValueExpression(I);
1856 const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
1857 assert(isa<CmpInst>(I) && "Expected a cmp instruction.");
1859 auto *CI = cast<CmpInst>(I);
1860 // See if our operands are equal to those of a previous predicate, and if so,
1861 // if it implies true or false.
1862 auto Op0 = lookupOperandLeader(CI->getOperand(0));
1863 auto Op1 = lookupOperandLeader(CI->getOperand(1));
1864 auto OurPredicate = CI->getPredicate();
1865 if (shouldSwapOperands(Op0, Op1)) {
1866 std::swap(Op0, Op1);
1867 OurPredicate = CI->getSwappedPredicate();
1870 // Avoid processing the same info twice.
1871 const PredicateBase *LastPredInfo = nullptr;
1872 // See if we know something about the comparison itself, like it is the target
1874 auto *CmpPI = PredInfo->getPredicateInfoFor(I);
1875 if (dyn_cast_or_null<PredicateAssume>(CmpPI))
1876 return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1879 // This condition does not depend on predicates, no need to add users
1880 if (CI->isTrueWhenEqual())
1881 return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1882 else if (CI->isFalseWhenEqual())
1883 return createConstantExpression(ConstantInt::getFalse(CI->getType()));
1886 // NOTE: Because we are comparing both operands here and below, and using
1887 // previous comparisons, we rely on fact that predicateinfo knows to mark
1888 // comparisons that use renamed operands as users of the earlier comparisons.
1889 // It is *not* enough to just mark predicateinfo renamed operands as users of
1890 // the earlier comparisons, because the *other* operand may have changed in a
1891 // previous iteration.
1894 // %b.0 = ssa.copy(%b)
1896 // icmp slt %c, %b.0
1898 // %c and %a may start out equal, and thus, the code below will say the second
1899 // %icmp is false. c may become equal to something else, and in that case the
1900 // %second icmp *must* be reexamined, but would not if only the renamed
1901 // %operands are considered users of the icmp.
1903 // *Currently* we only check one level of comparisons back, and only mark one
1904 // level back as touched when changes happen. If you modify this code to look
1905 // back farther through comparisons, you *must* mark the appropriate
1906 // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if
1907 // we know something just from the operands themselves
1909 // See if our operands have predicate info, so that we may be able to derive
1910 // something from a previous comparison.
1911 for (const auto &Op : CI->operands()) {
1912 auto *PI = PredInfo->getPredicateInfoFor(Op);
1913 if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) {
1914 if (PI == LastPredInfo)
1917 // In phi of ops cases, we may have predicate info that we are evaluating
1918 // in a different context.
1919 if (!DT->dominates(PBranch->To, getBlockForValue(I)))
1921 // TODO: Along the false edge, we may know more things too, like
1923 // same operands is false.
1924 // TODO: We only handle actual comparison conditions below, not
1926 auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition);
1929 auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0));
1930 auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1));
1931 auto BranchPredicate = BranchCond->getPredicate();
1932 if (shouldSwapOperands(BranchOp0, BranchOp1)) {
1933 std::swap(BranchOp0, BranchOp1);
1934 BranchPredicate = BranchCond->getSwappedPredicate();
1936 if (BranchOp0 == Op0 && BranchOp1 == Op1) {
1937 if (PBranch->TrueEdge) {
1938 // If we know the previous predicate is true and we are in the true
1939 // edge then we may be implied true or false.
1940 if (CmpInst::isImpliedTrueByMatchingCmp(BranchPredicate,
1942 addPredicateUsers(PI, I);
1943 return createConstantExpression(
1944 ConstantInt::getTrue(CI->getType()));
1947 if (CmpInst::isImpliedFalseByMatchingCmp(BranchPredicate,
1949 addPredicateUsers(PI, I);
1950 return createConstantExpression(
1951 ConstantInt::getFalse(CI->getType()));
1954 // Just handle the ne and eq cases, where if we have the same
1955 // operands, we may know something.
1956 if (BranchPredicate == OurPredicate) {
1957 addPredicateUsers(PI, I);
1958 // Same predicate, same ops,we know it was false, so this is false.
1959 return createConstantExpression(
1960 ConstantInt::getFalse(CI->getType()));
1961 } else if (BranchPredicate ==
1962 CmpInst::getInversePredicate(OurPredicate)) {
1963 addPredicateUsers(PI, I);
1964 // Inverse predicate, we know the other was false, so this is true.
1965 return createConstantExpression(
1966 ConstantInt::getTrue(CI->getType()));
1972 // Create expression will take care of simplifyCmpInst
1973 return createExpression(I);
1976 // Substitute and symbolize the value before value numbering.
1978 NewGVN::performSymbolicEvaluation(Value *V,
1979 SmallPtrSetImpl<Value *> &Visited) const {
1980 const Expression *E = nullptr;
1981 if (auto *C = dyn_cast<Constant>(V))
1982 E = createConstantExpression(C);
1983 else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
1984 E = createVariableExpression(V);
1986 // TODO: memory intrinsics.
1987 // TODO: Some day, we should do the forward propagation and reassociation
1988 // parts of the algorithm.
1989 auto *I = cast<Instruction>(V);
1990 switch (I->getOpcode()) {
1991 case Instruction::ExtractValue:
1992 case Instruction::InsertValue:
1993 E = performSymbolicAggrValueEvaluation(I);
1995 case Instruction::PHI: {
1996 SmallVector<ValPair, 3> Ops;
1997 auto *PN = cast<PHINode>(I);
1998 for (unsigned i = 0; i < PN->getNumOperands(); ++i)
1999 Ops.push_back({PN->getIncomingValue(i), PN->getIncomingBlock(i)});
2000 // Sort to ensure the invariant createPHIExpression requires is met.
2002 E = performSymbolicPHIEvaluation(Ops, I, getBlockForValue(I));
2004 case Instruction::Call:
2005 E = performSymbolicCallEvaluation(I);
2007 case Instruction::Store:
2008 E = performSymbolicStoreEvaluation(I);
2010 case Instruction::Load:
2011 E = performSymbolicLoadEvaluation(I);
2013 case Instruction::BitCast:
2014 E = createExpression(I);
2016 case Instruction::ICmp:
2017 case Instruction::FCmp:
2018 E = performSymbolicCmpEvaluation(I);
2020 case Instruction::Add:
2021 case Instruction::FAdd:
2022 case Instruction::Sub:
2023 case Instruction::FSub:
2024 case Instruction::Mul:
2025 case Instruction::FMul:
2026 case Instruction::UDiv:
2027 case Instruction::SDiv:
2028 case Instruction::FDiv:
2029 case Instruction::URem:
2030 case Instruction::SRem:
2031 case Instruction::FRem:
2032 case Instruction::Shl:
2033 case Instruction::LShr:
2034 case Instruction::AShr:
2035 case Instruction::And:
2036 case Instruction::Or:
2037 case Instruction::Xor:
2038 case Instruction::Trunc:
2039 case Instruction::ZExt:
2040 case Instruction::SExt:
2041 case Instruction::FPToUI:
2042 case Instruction::FPToSI:
2043 case Instruction::UIToFP:
2044 case Instruction::SIToFP:
2045 case Instruction::FPTrunc:
2046 case Instruction::FPExt:
2047 case Instruction::PtrToInt:
2048 case Instruction::IntToPtr:
2049 case Instruction::Select:
2050 case Instruction::ExtractElement:
2051 case Instruction::InsertElement:
2052 case Instruction::ShuffleVector:
2053 case Instruction::GetElementPtr:
2054 E = createExpression(I);
2063 // Look up a container in a map, and then call a function for each thing in the
2065 template <typename Map, typename KeyType, typename Func>
2066 void NewGVN::for_each_found(Map &M, const KeyType &Key, Func F) {
2067 const auto Result = M.find_as(Key);
2068 if (Result != M.end())
2069 for (typename Map::mapped_type::value_type Mapped : Result->second)
2073 // Look up a container of values/instructions in a map, and touch all the
2074 // instructions in the container. Then erase value from the map.
2075 template <typename Map, typename KeyType>
2076 void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2077 const auto Result = M.find_as(Key);
2078 if (Result != M.end()) {
2079 for (const typename Map::mapped_type::value_type Mapped : Result->second)
2080 TouchedInstructions.set(InstrToDFSNum(Mapped));
2085 void NewGVN::addAdditionalUsers(Value *To, Value *User) const {
2086 assert(User && To != User);
2087 if (isa<Instruction>(To))
2088 AdditionalUsers[To].insert(User);
2091 void NewGVN::markUsersTouched(Value *V) {
2092 // Now mark the users as touched.
2093 for (auto *User : V->users()) {
2094 assert(isa<Instruction>(User) && "Use of value not within an instruction?");
2095 TouchedInstructions.set(InstrToDFSNum(User));
2097 touchAndErase(AdditionalUsers, V);
2100 void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const {
2101 LLVM_DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");
2102 MemoryToUsers[To].insert(U);
2105 void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {
2106 TouchedInstructions.set(MemoryToDFSNum(MA));
2109 void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
2110 if (isa<MemoryUse>(MA))
2112 for (auto U : MA->users())
2113 TouchedInstructions.set(MemoryToDFSNum(U));
2114 touchAndErase(MemoryToUsers, MA);
2117 // Add I to the set of users of a given predicate.
2118 void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) const {
2119 // Don't add temporary instructions to the user lists.
2120 if (AllTempInstructions.count(I))
2123 if (auto *PBranch = dyn_cast<PredicateBranch>(PB))
2124 PredicateToUsers[PBranch->Condition].insert(I);
2125 else if (auto *PAssume = dyn_cast<PredicateBranch>(PB))
2126 PredicateToUsers[PAssume->Condition].insert(I);
2129 // Touch all the predicates that depend on this instruction.
2130 void NewGVN::markPredicateUsersTouched(Instruction *I) {
2131 touchAndErase(PredicateToUsers, I);
2134 // Mark users affected by a memory leader change.
2135 void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {
2136 for (auto M : CC->memory())
2137 markMemoryDefTouched(M);
2140 // Touch the instructions that need to be updated after a congruence class has a
2141 // leader change, and mark changed values.
2142 void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {
2143 for (auto M : *CC) {
2144 if (auto *I = dyn_cast<Instruction>(M))
2145 TouchedInstructions.set(InstrToDFSNum(I));
2146 LeaderChanges.insert(M);
2150 // Give a range of things that have instruction DFS numbers, this will return
2151 // the member of the range with the smallest dfs number.
2152 template <class T, class Range>
2153 T *NewGVN::getMinDFSOfRange(const Range &R) const {
2154 std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2155 for (const auto X : R) {
2156 auto DFSNum = InstrToDFSNum(X);
2157 if (DFSNum < MinDFS.second)
2158 MinDFS = {X, DFSNum};
2160 return MinDFS.first;
2163 // This function returns the MemoryAccess that should be the next leader of
2164 // congruence class CC, under the assumption that the current leader is going to
2166 const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {
2167 // TODO: If this ends up to slow, we can maintain a next memory leader like we
2168 // do for regular leaders.
2169 // Make sure there will be a leader to find.
2170 assert(!CC->definesNoMemory() && "Can't get next leader if there is none");
2171 if (CC->getStoreCount() > 0) {
2172 if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first))
2173 return getMemoryAccess(NL);
2174 // Find the store with the minimum DFS number.
2175 auto *V = getMinDFSOfRange<Value>(make_filter_range(
2176 *CC, [&](const Value *V) { return isa<StoreInst>(V); }));
2177 return getMemoryAccess(cast<StoreInst>(V));
2179 assert(CC->getStoreCount() == 0);
2181 // Given our assertion, hitting this part must mean
2182 // !OldClass->memory_empty()
2183 if (CC->memory_size() == 1)
2184 return *CC->memory_begin();
2185 return getMinDFSOfRange<const MemoryPhi>(CC->memory());
2188 // This function returns the next value leader of a congruence class, under the
2189 // assumption that the current leader is going away. This should end up being
2190 // the next most dominating member.
2191 Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {
2192 // We don't need to sort members if there is only 1, and we don't care about
2193 // sorting the TOP class because everything either gets out of it or is
2196 if (CC->size() == 1 || CC == TOPClass) {
2197 return *(CC->begin());
2198 } else if (CC->getNextLeader().first) {
2199 ++NumGVNAvoidedSortedLeaderChanges;
2200 return CC->getNextLeader().first;
2202 ++NumGVNSortedLeaderChanges;
2203 // NOTE: If this ends up to slow, we can maintain a dual structure for
2204 // member testing/insertion, or keep things mostly sorted, and sort only
2205 // here, or use SparseBitVector or ....
2206 return getMinDFSOfRange<Value>(*CC);
2210 // Move a MemoryAccess, currently in OldClass, to NewClass, including updates to
2211 // the memory members, etc for the move.
2213 // The invariants of this function are:
2215 // - I must be moving to NewClass from OldClass
2216 // - The StoreCount of OldClass and NewClass is expected to have been updated
2217 // for I already if it is a store.
2218 // - The OldClass memory leader has not been updated yet if I was the leader.
2219 void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,
2220 MemoryAccess *InstMA,
2221 CongruenceClass *OldClass,
2222 CongruenceClass *NewClass) {
2223 // If the leader is I, and we had a representative MemoryAccess, it should
2224 // be the MemoryAccess of OldClass.
2225 assert((!InstMA || !OldClass->getMemoryLeader() ||
2226 OldClass->getLeader() != I ||
2227 MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) ==
2228 MemoryAccessToClass.lookup(InstMA)) &&
2229 "Representative MemoryAccess mismatch");
2230 // First, see what happens to the new class
2231 if (!NewClass->getMemoryLeader()) {
2232 // Should be a new class, or a store becoming a leader of a new class.
2233 assert(NewClass->size() == 1 ||
2234 (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));
2235 NewClass->setMemoryLeader(InstMA);
2236 // Mark it touched if we didn't just create a singleton
2237 LLVM_DEBUG(dbgs() << "Memory class leader change for class "
2238 << NewClass->getID()
2239 << " due to new memory instruction becoming leader\n");
2240 markMemoryLeaderChangeTouched(NewClass);
2242 setMemoryClass(InstMA, NewClass);
2243 // Now, fixup the old class if necessary
2244 if (OldClass->getMemoryLeader() == InstMA) {
2245 if (!OldClass->definesNoMemory()) {
2246 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
2247 LLVM_DEBUG(dbgs() << "Memory class leader change for class "
2248 << OldClass->getID() << " to "
2249 << *OldClass->getMemoryLeader()
2250 << " due to removal of old leader " << *InstMA << "\n");
2251 markMemoryLeaderChangeTouched(OldClass);
2253 OldClass->setMemoryLeader(nullptr);
2257 // Move a value, currently in OldClass, to be part of NewClass
2258 // Update OldClass and NewClass for the move (including changing leaders, etc).
2259 void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,
2260 CongruenceClass *OldClass,
2261 CongruenceClass *NewClass) {
2262 if (I == OldClass->getNextLeader().first)
2263 OldClass->resetNextLeader();
2266 NewClass->insert(I);
2268 if (NewClass->getLeader() != I)
2269 NewClass->addPossibleNextLeader({I, InstrToDFSNum(I)});
2270 // Handle our special casing of stores.
2271 if (auto *SI = dyn_cast<StoreInst>(I)) {
2272 OldClass->decStoreCount();
2273 // Okay, so when do we want to make a store a leader of a class?
2274 // If we have a store defined by an earlier load, we want the earlier load
2275 // to lead the class.
2276 // If we have a store defined by something else, we want the store to lead
2277 // the class so everything else gets the "something else" as a value.
2278 // If we have a store as the single member of the class, we want the store
2280 if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) {
2281 // If it's a store expression we are using, it means we are not equivalent
2282 // to something earlier.
2283 if (auto *SE = dyn_cast<StoreExpression>(E)) {
2284 NewClass->setStoredValue(SE->getStoredValue());
2285 markValueLeaderChangeTouched(NewClass);
2286 // Shift the new class leader to be the store
2287 LLVM_DEBUG(dbgs() << "Changing leader of congruence class "
2288 << NewClass->getID() << " from "
2289 << *NewClass->getLeader() << " to " << *SI
2290 << " because store joined class\n");
2291 // If we changed the leader, we have to mark it changed because we don't
2292 // know what it will do to symbolic evaluation.
2293 NewClass->setLeader(SI);
2295 // We rely on the code below handling the MemoryAccess change.
2297 NewClass->incStoreCount();
2299 // True if there is no memory instructions left in a class that had memory
2300 // instructions before.
2302 // If it's not a memory use, set the MemoryAccess equivalence
2303 auto *InstMA = dyn_cast_or_null<MemoryDef>(getMemoryAccess(I));
2305 moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);
2306 ValueToClass[I] = NewClass;
2307 // See if we destroyed the class or need to swap leaders.
2308 if (OldClass->empty() && OldClass != TOPClass) {
2309 if (OldClass->getDefiningExpr()) {
2310 LLVM_DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()
2311 << " from table\n");
2312 // We erase it as an exact expression to make sure we don't just erase an
2314 auto Iter = ExpressionToClass.find_as(
2315 ExactEqualsExpression(*OldClass->getDefiningExpr()));
2316 if (Iter != ExpressionToClass.end())
2317 ExpressionToClass.erase(Iter);
2318 #ifdef EXPENSIVE_CHECKS
2320 (*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) &&
2321 "We erased the expression we just inserted, which should not happen");
2324 } else if (OldClass->getLeader() == I) {
2325 // When the leader changes, the value numbering of
2326 // everything may change due to symbolization changes, so we need to
2328 LLVM_DEBUG(dbgs() << "Value class leader change for class "
2329 << OldClass->getID() << "\n");
2330 ++NumGVNLeaderChanges;
2331 // Destroy the stored value if there are no more stores to represent it.
2332 // Note that this is basically clean up for the expression removal that
2333 // happens below. If we remove stores from a class, we may leave it as a
2334 // class of equivalent memory phis.
2335 if (OldClass->getStoreCount() == 0) {
2336 if (OldClass->getStoredValue())
2337 OldClass->setStoredValue(nullptr);
2339 OldClass->setLeader(getNextValueLeader(OldClass));
2340 OldClass->resetNextLeader();
2341 markValueLeaderChangeTouched(OldClass);
2345 // For a given expression, mark the phi of ops instructions that could have
2346 // changed as a result.
2347 void NewGVN::markPhiOfOpsChanged(const Expression *E) {
2348 touchAndErase(ExpressionToPhiOfOps, E);
2351 // Perform congruence finding on a given value numbering expression.
2352 void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
2353 // This is guaranteed to return something, since it will at least find
2356 CongruenceClass *IClass = ValueToClass.lookup(I);
2357 assert(IClass && "Should have found a IClass");
2358 // Dead classes should have been eliminated from the mapping.
2359 assert(!IClass->isDead() && "Found a dead class");
2361 CongruenceClass *EClass = nullptr;
2362 if (const auto *VE = dyn_cast<VariableExpression>(E)) {
2363 EClass = ValueToClass.lookup(VE->getVariableValue());
2364 } else if (isa<DeadExpression>(E)) {
2368 auto lookupResult = ExpressionToClass.insert({E, nullptr});
2370 // If it's not in the value table, create a new congruence class.
2371 if (lookupResult.second) {
2372 CongruenceClass *NewClass = createCongruenceClass(nullptr, E);
2373 auto place = lookupResult.first;
2374 place->second = NewClass;
2376 // Constants and variables should always be made the leader.
2377 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2378 NewClass->setLeader(CE->getConstantValue());
2379 } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {
2380 StoreInst *SI = SE->getStoreInst();
2381 NewClass->setLeader(SI);
2382 NewClass->setStoredValue(SE->getStoredValue());
2383 // The RepMemoryAccess field will be filled in properly by the
2384 // moveValueToNewCongruenceClass call.
2386 NewClass->setLeader(I);
2388 assert(!isa<VariableExpression>(E) &&
2389 "VariableExpression should have been handled already");
2392 LLVM_DEBUG(dbgs() << "Created new congruence class for " << *I
2393 << " using expression " << *E << " at "
2394 << NewClass->getID() << " and leader "
2395 << *(NewClass->getLeader()));
2396 if (NewClass->getStoredValue())
2397 LLVM_DEBUG(dbgs() << " and stored value "
2398 << *(NewClass->getStoredValue()));
2399 LLVM_DEBUG(dbgs() << "\n");
2401 EClass = lookupResult.first->second;
2402 if (isa<ConstantExpression>(E))
2403 assert((isa<Constant>(EClass->getLeader()) ||
2404 (EClass->getStoredValue() &&
2405 isa<Constant>(EClass->getStoredValue()))) &&
2406 "Any class with a constant expression should have a "
2409 assert(EClass && "Somehow don't have an eclass");
2411 assert(!EClass->isDead() && "We accidentally looked up a dead class");
2414 bool ClassChanged = IClass != EClass;
2415 bool LeaderChanged = LeaderChanges.erase(I);
2416 if (ClassChanged || LeaderChanged) {
2417 LLVM_DEBUG(dbgs() << "New class " << EClass->getID() << " for expression "
2420 moveValueToNewCongruenceClass(I, E, IClass, EClass);
2421 markPhiOfOpsChanged(E);
2424 markUsersTouched(I);
2425 if (MemoryAccess *MA = getMemoryAccess(I))
2426 markMemoryUsersTouched(MA);
2427 if (auto *CI = dyn_cast<CmpInst>(I))
2428 markPredicateUsersTouched(CI);
2430 // If we changed the class of the store, we want to ensure nothing finds the
2431 // old store expression. In particular, loads do not compare against stored
2432 // value, so they will find old store expressions (and associated class
2433 // mappings) if we leave them in the table.
2434 if (ClassChanged && isa<StoreInst>(I)) {
2435 auto *OldE = ValueToExpression.lookup(I);
2436 // It could just be that the old class died. We don't want to erase it if we
2437 // just moved classes.
2438 if (OldE && isa<StoreExpression>(OldE) && *E != *OldE) {
2439 // Erase this as an exact expression to ensure we don't erase expressions
2440 // equivalent to it.
2441 auto Iter = ExpressionToClass.find_as(ExactEqualsExpression(*OldE));
2442 if (Iter != ExpressionToClass.end())
2443 ExpressionToClass.erase(Iter);
2446 ValueToExpression[I] = E;
2449 // Process the fact that Edge (from, to) is reachable, including marking
2450 // any newly reachable blocks and instructions for processing.
2451 void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
2452 // Check if the Edge was reachable before.
2453 if (ReachableEdges.insert({From, To}).second) {
2454 // If this block wasn't reachable before, all instructions are touched.
2455 if (ReachableBlocks.insert(To).second) {
2456 LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)
2457 << " marked reachable\n");
2458 const auto &InstRange = BlockInstRange.lookup(To);
2459 TouchedInstructions.set(InstRange.first, InstRange.second);
2461 LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)
2462 << " was reachable, but new edge {"
2463 << getBlockName(From) << "," << getBlockName(To)
2464 << "} to it found\n");
2466 // We've made an edge reachable to an existing block, which may
2467 // impact predicates. Otherwise, only mark the phi nodes as touched, as
2468 // they are the only thing that depend on new edges. Anything using their
2469 // values will get propagated to if necessary.
2470 if (MemoryAccess *MemPhi = getMemoryAccess(To))
2471 TouchedInstructions.set(InstrToDFSNum(MemPhi));
2473 // FIXME: We should just add a union op on a Bitvector and
2474 // SparseBitVector. We can do it word by word faster than we are doing it
2476 for (auto InstNum : RevisitOnReachabilityChange[To])
2477 TouchedInstructions.set(InstNum);
2482 // Given a predicate condition (from a switch, cmp, or whatever) and a block,
2483 // see if we know some constant value for it already.
2484 Value *NewGVN::findConditionEquivalence(Value *Cond) const {
2485 auto Result = lookupOperandLeader(Cond);
2486 return isa<Constant>(Result) ? Result : nullptr;
2489 // Process the outgoing edges of a block for reachability.
2490 void NewGVN::processOutgoingEdges(Instruction *TI, BasicBlock *B) {
2491 // Evaluate reachability of terminator instruction.
2493 if ((BR = dyn_cast<BranchInst>(TI)) && BR->isConditional()) {
2494 Value *Cond = BR->getCondition();
2495 Value *CondEvaluated = findConditionEquivalence(Cond);
2496 if (!CondEvaluated) {
2497 if (auto *I = dyn_cast<Instruction>(Cond)) {
2498 const Expression *E = createExpression(I);
2499 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2500 CondEvaluated = CE->getConstantValue();
2502 } else if (isa<ConstantInt>(Cond)) {
2503 CondEvaluated = Cond;
2507 BasicBlock *TrueSucc = BR->getSuccessor(0);
2508 BasicBlock *FalseSucc = BR->getSuccessor(1);
2509 if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) {
2511 LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI
2512 << " evaluated to true\n");
2513 updateReachableEdge(B, TrueSucc);
2514 } else if (CI->isZero()) {
2515 LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI
2516 << " evaluated to false\n");
2517 updateReachableEdge(B, FalseSucc);
2520 updateReachableEdge(B, TrueSucc);
2521 updateReachableEdge(B, FalseSucc);
2523 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
2524 // For switches, propagate the case values into the case
2527 // Remember how many outgoing edges there are to every successor.
2528 SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges;
2530 Value *SwitchCond = SI->getCondition();
2531 Value *CondEvaluated = findConditionEquivalence(SwitchCond);
2532 // See if we were able to turn this switch statement into a constant.
2533 if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) {
2534 auto *CondVal = cast<ConstantInt>(CondEvaluated);
2535 // We should be able to get case value for this.
2536 auto Case = *SI->findCaseValue(CondVal);
2537 if (Case.getCaseSuccessor() == SI->getDefaultDest()) {
2538 // We proved the value is outside of the range of the case.
2539 // We can't do anything other than mark the default dest as reachable,
2541 updateReachableEdge(B, SI->getDefaultDest());
2544 // Now get where it goes and mark it reachable.
2545 BasicBlock *TargetBlock = Case.getCaseSuccessor();
2546 updateReachableEdge(B, TargetBlock);
2548 for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; ++i) {
2549 BasicBlock *TargetBlock = SI->getSuccessor(i);
2550 ++SwitchEdges[TargetBlock];
2551 updateReachableEdge(B, TargetBlock);
2555 // Otherwise this is either unconditional, or a type we have no
2556 // idea about. Just mark successors as reachable.
2557 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
2558 BasicBlock *TargetBlock = TI->getSuccessor(i);
2559 updateReachableEdge(B, TargetBlock);
2562 // This also may be a memory defining terminator, in which case, set it
2563 // equivalent only to itself.
2565 auto *MA = getMemoryAccess(TI);
2566 if (MA && !isa<MemoryUse>(MA)) {
2567 auto *CC = ensureLeaderOfMemoryClass(MA);
2568 if (setMemoryClass(MA, CC))
2569 markMemoryUsersTouched(MA);
2574 // Remove the PHI of Ops PHI for I
2575 void NewGVN::removePhiOfOps(Instruction *I, PHINode *PHITemp) {
2576 InstrDFS.erase(PHITemp);
2577 // It's still a temp instruction. We keep it in the array so it gets erased.
2578 // However, it's no longer used by I, or in the block
2579 TempToBlock.erase(PHITemp);
2580 RealToTemp.erase(I);
2581 // We don't remove the users from the phi node uses. This wastes a little
2582 // time, but such is life. We could use two sets to track which were there
2583 // are the start of NewGVN, and which were added, but right nowt he cost of
2584 // tracking is more than the cost of checking for more phi of ops.
2587 // Add PHI Op in BB as a PHI of operations version of ExistingValue.
2588 void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB,
2589 Instruction *ExistingValue) {
2590 InstrDFS[Op] = InstrToDFSNum(ExistingValue);
2591 AllTempInstructions.insert(Op);
2592 TempToBlock[Op] = BB;
2593 RealToTemp[ExistingValue] = Op;
2594 // Add all users to phi node use, as they are now uses of the phi of ops phis
2595 // and may themselves be phi of ops.
2596 for (auto *U : ExistingValue->users())
2597 if (auto *UI = dyn_cast<Instruction>(U))
2598 PHINodeUses.insert(UI);
2601 static bool okayForPHIOfOps(const Instruction *I) {
2602 if (!EnablePhiOfOps)
2604 return isa<BinaryOperator>(I) || isa<SelectInst>(I) || isa<CmpInst>(I) ||
2608 bool NewGVN::OpIsSafeForPHIOfOpsHelper(
2609 Value *V, const BasicBlock *PHIBlock,
2610 SmallPtrSetImpl<const Value *> &Visited,
2611 SmallVectorImpl<Instruction *> &Worklist) {
2613 if (!isa<Instruction>(V))
2615 auto OISIt = OpSafeForPHIOfOps.find(V);
2616 if (OISIt != OpSafeForPHIOfOps.end())
2617 return OISIt->second;
2619 // Keep walking until we either dominate the phi block, or hit a phi, or run
2620 // out of things to check.
2621 if (DT->properlyDominates(getBlockForValue(V), PHIBlock)) {
2622 OpSafeForPHIOfOps.insert({V, true});
2625 // PHI in the same block.
2626 if (isa<PHINode>(V) && getBlockForValue(V) == PHIBlock) {
2627 OpSafeForPHIOfOps.insert({V, false});
2631 auto *OrigI = cast<Instruction>(V);
2632 for (auto *Op : OrigI->operand_values()) {
2633 if (!isa<Instruction>(Op))
2635 // Stop now if we find an unsafe operand.
2636 auto OISIt = OpSafeForPHIOfOps.find(OrigI);
2637 if (OISIt != OpSafeForPHIOfOps.end()) {
2638 if (!OISIt->second) {
2639 OpSafeForPHIOfOps.insert({V, false});
2644 if (!Visited.insert(Op).second)
2646 Worklist.push_back(cast<Instruction>(Op));
2651 // Return true if this operand will be safe to use for phi of ops.
2653 // The reason some operands are unsafe is that we are not trying to recursively
2654 // translate everything back through phi nodes. We actually expect some lookups
2655 // of expressions to fail. In particular, a lookup where the expression cannot
2656 // exist in the predecessor. This is true even if the expression, as shown, can
2657 // be determined to be constant.
2658 bool NewGVN::OpIsSafeForPHIOfOps(Value *V, const BasicBlock *PHIBlock,
2659 SmallPtrSetImpl<const Value *> &Visited) {
2660 SmallVector<Instruction *, 4> Worklist;
2661 if (!OpIsSafeForPHIOfOpsHelper(V, PHIBlock, Visited, Worklist))
2663 while (!Worklist.empty()) {
2664 auto *I = Worklist.pop_back_val();
2665 if (!OpIsSafeForPHIOfOpsHelper(I, PHIBlock, Visited, Worklist))
2668 OpSafeForPHIOfOps.insert({V, true});
2672 // Try to find a leader for instruction TransInst, which is a phi translated
2673 // version of something in our original program. Visited is used to ensure we
2674 // don't infinite loop during translations of cycles. OrigInst is the
2675 // instruction in the original program, and PredBB is the predecessor we
2676 // translated it through.
2677 Value *NewGVN::findLeaderForInst(Instruction *TransInst,
2678 SmallPtrSetImpl<Value *> &Visited,
2679 MemoryAccess *MemAccess, Instruction *OrigInst,
2680 BasicBlock *PredBB) {
2681 unsigned IDFSNum = InstrToDFSNum(OrigInst);
2682 // Make sure it's marked as a temporary instruction.
2683 AllTempInstructions.insert(TransInst);
2684 // and make sure anything that tries to add it's DFS number is
2685 // redirected to the instruction we are making a phi of ops
2687 TempToBlock.insert({TransInst, PredBB});
2688 InstrDFS.insert({TransInst, IDFSNum});
2690 const Expression *E = performSymbolicEvaluation(TransInst, Visited);
2691 InstrDFS.erase(TransInst);
2692 AllTempInstructions.erase(TransInst);
2693 TempToBlock.erase(TransInst);
2695 TempToMemory.erase(TransInst);
2698 auto *FoundVal = findPHIOfOpsLeader(E, OrigInst, PredBB);
2700 ExpressionToPhiOfOps[E].insert(OrigInst);
2701 LLVM_DEBUG(dbgs() << "Cannot find phi of ops operand for " << *TransInst
2702 << " in block " << getBlockName(PredBB) << "\n");
2705 if (auto *SI = dyn_cast<StoreInst>(FoundVal))
2706 FoundVal = SI->getValueOperand();
2710 // When we see an instruction that is an op of phis, generate the equivalent phi
2713 NewGVN::makePossiblePHIOfOps(Instruction *I,
2714 SmallPtrSetImpl<Value *> &Visited) {
2715 if (!okayForPHIOfOps(I))
2718 if (!Visited.insert(I).second)
2720 // For now, we require the instruction be cycle free because we don't
2721 // *always* create a phi of ops for instructions that could be done as phi
2722 // of ops, we only do it if we think it is useful. If we did do it all the
2723 // time, we could remove the cycle free check.
2724 if (!isCycleFree(I))
2727 SmallPtrSet<const Value *, 8> ProcessedPHIs;
2728 // TODO: We don't do phi translation on memory accesses because it's
2729 // complicated. For a load, we'd need to be able to simulate a new memoryuse,
2730 // which we don't have a good way of doing ATM.
2731 auto *MemAccess = getMemoryAccess(I);
2732 // If the memory operation is defined by a memory operation this block that
2733 // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi
2734 // can't help, as it would still be killed by that memory operation.
2735 if (MemAccess && !isa<MemoryPhi>(MemAccess->getDefiningAccess()) &&
2736 MemAccess->getDefiningAccess()->getBlock() == I->getParent())
2739 // Convert op of phis to phi of ops
2740 SmallPtrSet<const Value *, 10> VisitedOps;
2741 SmallVector<Value *, 4> Ops(I->operand_values());
2742 BasicBlock *SamePHIBlock = nullptr;
2743 PHINode *OpPHI = nullptr;
2744 if (!DebugCounter::shouldExecute(PHIOfOpsCounter))
2746 for (auto *Op : Ops) {
2747 if (!isa<PHINode>(Op)) {
2748 auto *ValuePHI = RealToTemp.lookup(Op);
2751 LLVM_DEBUG(dbgs() << "Found possible dependent phi of ops\n");
2754 OpPHI = cast<PHINode>(Op);
2755 if (!SamePHIBlock) {
2756 SamePHIBlock = getBlockForValue(OpPHI);
2757 } else if (SamePHIBlock != getBlockForValue(OpPHI)) {
2760 << "PHIs for operands are not all in the same block, aborting\n");
2763 // No point in doing this for one-operand phis.
2764 if (OpPHI->getNumOperands() == 1) {
2773 SmallVector<ValPair, 4> PHIOps;
2774 SmallPtrSet<Value *, 4> Deps;
2775 auto *PHIBlock = getBlockForValue(OpPHI);
2776 RevisitOnReachabilityChange[PHIBlock].reset(InstrToDFSNum(I));
2777 for (unsigned PredNum = 0; PredNum < OpPHI->getNumOperands(); ++PredNum) {
2778 auto *PredBB = OpPHI->getIncomingBlock(PredNum);
2779 Value *FoundVal = nullptr;
2780 SmallPtrSet<Value *, 4> CurrentDeps;
2781 // We could just skip unreachable edges entirely but it's tricky to do
2782 // with rewriting existing phi nodes.
2783 if (ReachableEdges.count({PredBB, PHIBlock})) {
2784 // Clone the instruction, create an expression from it that is
2785 // translated back into the predecessor, and see if we have a leader.
2786 Instruction *ValueOp = I->clone();
2788 TempToMemory.insert({ValueOp, MemAccess});
2789 bool SafeForPHIOfOps = true;
2791 for (auto &Op : ValueOp->operands()) {
2792 auto *OrigOp = &*Op;
2793 // When these operand changes, it could change whether there is a
2794 // leader for us or not, so we have to add additional users.
2795 if (isa<PHINode>(Op)) {
2796 Op = Op->DoPHITranslation(PHIBlock, PredBB);
2797 if (Op != OrigOp && Op != I)
2798 CurrentDeps.insert(Op);
2799 } else if (auto *ValuePHI = RealToTemp.lookup(Op)) {
2800 if (getBlockForValue(ValuePHI) == PHIBlock)
2801 Op = ValuePHI->getIncomingValueForBlock(PredBB);
2803 // If we phi-translated the op, it must be safe.
2806 (Op != OrigOp || OpIsSafeForPHIOfOps(Op, PHIBlock, VisitedOps));
2808 // FIXME: For those things that are not safe we could generate
2809 // expressions all the way down, and see if this comes out to a
2810 // constant. For anything where that is true, and unsafe, we should
2811 // have made a phi-of-ops (or value numbered it equivalent to something)
2812 // for the pieces already.
2813 FoundVal = !SafeForPHIOfOps ? nullptr
2814 : findLeaderForInst(ValueOp, Visited,
2815 MemAccess, I, PredBB);
2816 ValueOp->deleteValue();
2818 // We failed to find a leader for the current ValueOp, but this might
2819 // change in case of the translated operands change.
2820 if (SafeForPHIOfOps)
2821 for (auto Dep : CurrentDeps)
2822 addAdditionalUsers(Dep, I);
2826 Deps.insert(CurrentDeps.begin(), CurrentDeps.end());
2828 LLVM_DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "
2829 << getBlockName(PredBB)
2830 << " because the block is unreachable\n");
2831 FoundVal = UndefValue::get(I->getType());
2832 RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));
2835 PHIOps.push_back({FoundVal, PredBB});
2836 LLVM_DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "
2837 << getBlockName(PredBB) << "\n");
2839 for (auto Dep : Deps)
2840 addAdditionalUsers(Dep, I);
2842 auto *E = performSymbolicPHIEvaluation(PHIOps, I, PHIBlock);
2843 if (isa<ConstantExpression>(E) || isa<VariableExpression>(E)) {
2846 << "Not creating real PHI of ops because it simplified to existing "
2847 "value or constant\n");
2850 auto *ValuePHI = RealToTemp.lookup(I);
2851 bool NewPHI = false;
2854 PHINode::Create(I->getType(), OpPHI->getNumOperands(), "phiofops");
2855 addPhiOfOps(ValuePHI, PHIBlock, I);
2857 NumGVNPHIOfOpsCreated++;
2860 for (auto PHIOp : PHIOps)
2861 ValuePHI->addIncoming(PHIOp.first, PHIOp.second);
2863 TempToBlock[ValuePHI] = PHIBlock;
2865 for (auto PHIOp : PHIOps) {
2866 ValuePHI->setIncomingValue(i, PHIOp.first);
2867 ValuePHI->setIncomingBlock(i, PHIOp.second);
2871 RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));
2872 LLVM_DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *I
2878 // The algorithm initially places the values of the routine in the TOP
2879 // congruence class. The leader of TOP is the undetermined value `undef`.
2880 // When the algorithm has finished, values still in TOP are unreachable.
2881 void NewGVN::initializeCongruenceClasses(Function &F) {
2882 NextCongruenceNum = 0;
2884 // Note that even though we use the live on entry def as a representative
2885 // MemoryAccess, it is *not* the same as the actual live on entry def. We
2886 // have no real equivalemnt to undef for MemoryAccesses, and so we really
2887 // should be checking whether the MemoryAccess is top if we want to know if it
2888 // is equivalent to everything. Otherwise, what this really signifies is that
2889 // the access "it reaches all the way back to the beginning of the function"
2891 // Initialize all other instructions to be in TOP class.
2892 TOPClass = createCongruenceClass(nullptr, nullptr);
2893 TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());
2894 // The live on entry def gets put into it's own class
2895 MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =
2896 createMemoryClass(MSSA->getLiveOnEntryDef());
2898 for (auto DTN : nodes(DT)) {
2899 BasicBlock *BB = DTN->getBlock();
2900 // All MemoryAccesses are equivalent to live on entry to start. They must
2901 // be initialized to something so that initial changes are noticed. For
2902 // the maximal answer, we initialize them all to be the same as
2904 auto *MemoryBlockDefs = MSSA->getBlockDefs(BB);
2905 if (MemoryBlockDefs)
2906 for (const auto &Def : *MemoryBlockDefs) {
2907 MemoryAccessToClass[&Def] = TOPClass;
2908 auto *MD = dyn_cast<MemoryDef>(&Def);
2909 // Insert the memory phis into the member list.
2911 const MemoryPhi *MP = cast<MemoryPhi>(&Def);
2912 TOPClass->memory_insert(MP);
2913 MemoryPhiState.insert({MP, MPS_TOP});
2916 if (MD && isa<StoreInst>(MD->getMemoryInst()))
2917 TOPClass->incStoreCount();
2920 // FIXME: This is trying to discover which instructions are uses of phi
2921 // nodes. We should move this into one of the myriad of places that walk
2922 // all the operands already.
2923 for (auto &I : *BB) {
2924 if (isa<PHINode>(&I))
2925 for (auto *U : I.users())
2926 if (auto *UInst = dyn_cast<Instruction>(U))
2927 if (InstrToDFSNum(UInst) != 0 && okayForPHIOfOps(UInst))
2928 PHINodeUses.insert(UInst);
2929 // Don't insert void terminators into the class. We don't value number
2930 // them, and they just end up sitting in TOP.
2931 if (I.isTerminator() && I.getType()->isVoidTy())
2933 TOPClass->insert(&I);
2934 ValueToClass[&I] = TOPClass;
2938 // Initialize arguments to be in their own unique congruence classes
2939 for (auto &FA : F.args())
2940 createSingletonCongruenceClass(&FA);
2943 void NewGVN::cleanupTables() {
2944 for (unsigned i = 0, e = CongruenceClasses.size(); i != e; ++i) {
2945 LLVM_DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->getID()
2946 << " has " << CongruenceClasses[i]->size()
2948 // Make sure we delete the congruence class (probably worth switching to
2949 // a unique_ptr at some point.
2950 delete CongruenceClasses[i];
2951 CongruenceClasses[i] = nullptr;
2954 // Destroy the value expressions
2955 SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(),
2956 AllTempInstructions.end());
2957 AllTempInstructions.clear();
2959 // We have to drop all references for everything first, so there are no uses
2960 // left as we delete them.
2961 for (auto *I : TempInst) {
2962 I->dropAllReferences();
2965 while (!TempInst.empty()) {
2966 auto *I = TempInst.back();
2967 TempInst.pop_back();
2971 ValueToClass.clear();
2972 ArgRecycler.clear(ExpressionAllocator);
2973 ExpressionAllocator.Reset();
2974 CongruenceClasses.clear();
2975 ExpressionToClass.clear();
2976 ValueToExpression.clear();
2978 AdditionalUsers.clear();
2979 ExpressionToPhiOfOps.clear();
2980 TempToBlock.clear();
2981 TempToMemory.clear();
2982 PHINodeUses.clear();
2983 OpSafeForPHIOfOps.clear();
2984 ReachableBlocks.clear();
2985 ReachableEdges.clear();
2987 ProcessedCount.clear();
2990 InstructionsToErase.clear();
2992 BlockInstRange.clear();
2993 TouchedInstructions.clear();
2994 MemoryAccessToClass.clear();
2995 PredicateToUsers.clear();
2996 MemoryToUsers.clear();
2997 RevisitOnReachabilityChange.clear();
3000 // Assign local DFS number mapping to instructions, and leave space for Value
3002 std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
3004 unsigned End = Start;
3005 if (MemoryAccess *MemPhi = getMemoryAccess(B)) {
3006 InstrDFS[MemPhi] = End++;
3007 DFSToInstr.emplace_back(MemPhi);
3010 // Then the real block goes next.
3011 for (auto &I : *B) {
3012 // There's no need to call isInstructionTriviallyDead more than once on
3013 // an instruction. Therefore, once we know that an instruction is dead
3014 // we change its DFS number so that it doesn't get value numbered.
3015 if (isInstructionTriviallyDead(&I, TLI)) {
3017 LLVM_DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");
3018 markInstructionForDeletion(&I);
3021 if (isa<PHINode>(&I))
3022 RevisitOnReachabilityChange[B].set(End);
3023 InstrDFS[&I] = End++;
3024 DFSToInstr.emplace_back(&I);
3027 // All of the range functions taken half-open ranges (open on the end side).
3028 // So we do not subtract one from count, because at this point it is one
3029 // greater than the last instruction.
3030 return std::make_pair(Start, End);
3033 void NewGVN::updateProcessedCount(const Value *V) {
3035 if (ProcessedCount.count(V) == 0) {
3036 ProcessedCount.insert({V, 1});
3038 ++ProcessedCount[V];
3039 assert(ProcessedCount[V] < 100 &&
3040 "Seem to have processed the same Value a lot");
3045 // Evaluate MemoryPhi nodes symbolically, just like PHI nodes
3046 void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
3047 // If all the arguments are the same, the MemoryPhi has the same value as the
3048 // argument. Filter out unreachable blocks and self phis from our operands.
3049 // TODO: We could do cycle-checking on the memory phis to allow valueizing for
3050 // self-phi checking.
3051 const BasicBlock *PHIBlock = MP->getBlock();
3052 auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {
3053 return cast<MemoryAccess>(U) != MP &&
3054 !isMemoryAccessTOP(cast<MemoryAccess>(U)) &&
3055 ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock});
3057 // If all that is left is nothing, our memoryphi is undef. We keep it as
3058 // InitialClass. Note: The only case this should happen is if we have at
3059 // least one self-argument.
3060 if (Filtered.begin() == Filtered.end()) {
3061 if (setMemoryClass(MP, TOPClass))
3062 markMemoryUsersTouched(MP);
3066 // Transform the remaining operands into operand leaders.
3067 // FIXME: mapped_iterator should have a range version.
3068 auto LookupFunc = [&](const Use &U) {
3069 return lookupMemoryLeader(cast<MemoryAccess>(U));
3071 auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);
3072 auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);
3074 // and now check if all the elements are equal.
3075 // Sadly, we can't use std::equals since these are random access iterators.
3076 const auto *AllSameValue = *MappedBegin;
3078 bool AllEqual = std::all_of(
3079 MappedBegin, MappedEnd,
3080 [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });
3083 LLVM_DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue
3086 LLVM_DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
3087 // If it's equal to something, it's in that class. Otherwise, it has to be in
3088 // a class where it is the leader (other things may be equivalent to it, but
3089 // it needs to start off in its own class, which means it must have been the
3090 // leader, and it can't have stopped being the leader because it was never
3092 CongruenceClass *CC =
3093 AllEqual ? getMemoryClass(AllSameValue) : ensureLeaderOfMemoryClass(MP);
3094 auto OldState = MemoryPhiState.lookup(MP);
3095 assert(OldState != MPS_Invalid && "Invalid memory phi state");
3096 auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique;
3097 MemoryPhiState[MP] = NewState;
3098 if (setMemoryClass(MP, CC) || OldState != NewState)
3099 markMemoryUsersTouched(MP);
3102 // Value number a single instruction, symbolically evaluating, performing
3103 // congruence finding, and updating mappings.
3104 void NewGVN::valueNumberInstruction(Instruction *I) {
3105 LLVM_DEBUG(dbgs() << "Processing instruction " << *I << "\n");
3106 if (!I->isTerminator()) {
3107 const Expression *Symbolized = nullptr;
3108 SmallPtrSet<Value *, 2> Visited;
3109 if (DebugCounter::shouldExecute(VNCounter)) {
3110 Symbolized = performSymbolicEvaluation(I, Visited);
3111 // Make a phi of ops if necessary
3112 if (Symbolized && !isa<ConstantExpression>(Symbolized) &&
3113 !isa<VariableExpression>(Symbolized) && PHINodeUses.count(I)) {
3114 auto *PHIE = makePossiblePHIOfOps(I, Visited);
3115 // If we created a phi of ops, use it.
3116 // If we couldn't create one, make sure we don't leave one lying around
3119 } else if (auto *Op = RealToTemp.lookup(I)) {
3120 removePhiOfOps(I, Op);
3124 // Mark the instruction as unused so we don't value number it again.
3127 // If we couldn't come up with a symbolic expression, use the unknown
3129 if (Symbolized == nullptr)
3130 Symbolized = createUnknownExpression(I);
3131 performCongruenceFinding(I, Symbolized);
3133 // Handle terminators that return values. All of them produce values we
3134 // don't currently understand. We don't place non-value producing
3135 // terminators in a class.
3136 if (!I->getType()->isVoidTy()) {
3137 auto *Symbolized = createUnknownExpression(I);
3138 performCongruenceFinding(I, Symbolized);
3140 processOutgoingEdges(I, I->getParent());
3144 // Check if there is a path, using single or equal argument phi nodes, from
3146 bool NewGVN::singleReachablePHIPath(
3147 SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First,
3148 const MemoryAccess *Second) const {
3149 if (First == Second)
3151 if (MSSA->isLiveOnEntryDef(First))
3154 // This is not perfect, but as we're just verifying here, we can live with
3155 // the loss of precision. The real solution would be that of doing strongly
3156 // connected component finding in this routine, and it's probably not worth
3157 // the complexity for the time being. So, we just keep a set of visited
3158 // MemoryAccess and return true when we hit a cycle.
3159 if (Visited.count(First))
3161 Visited.insert(First);
3163 const auto *EndDef = First;
3164 for (auto *ChainDef : optimized_def_chain(First)) {
3165 if (ChainDef == Second)
3167 if (MSSA->isLiveOnEntryDef(ChainDef))
3171 auto *MP = cast<MemoryPhi>(EndDef);
3172 auto ReachableOperandPred = [&](const Use &U) {
3173 return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()});
3175 auto FilteredPhiArgs =
3176 make_filter_range(MP->operands(), ReachableOperandPred);
3177 SmallVector<const Value *, 32> OperandList;
3178 llvm::copy(FilteredPhiArgs, std::back_inserter(OperandList));
3179 bool Okay = is_splat(OperandList);
3181 return singleReachablePHIPath(Visited, cast<MemoryAccess>(OperandList[0]),
3186 // Verify the that the memory equivalence table makes sense relative to the
3187 // congruence classes. Note that this checking is not perfect, and is currently
3188 // subject to very rare false negatives. It is only useful for
3189 // testing/debugging.
3190 void NewGVN::verifyMemoryCongruency() const {
3192 // Verify that the memory table equivalence and memory member set match
3193 for (const auto *CC : CongruenceClasses) {
3194 if (CC == TOPClass || CC->isDead())
3196 if (CC->getStoreCount() != 0) {
3197 assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&
3198 "Any class with a store as a leader should have a "
3199 "representative stored value");
3200 assert(CC->getMemoryLeader() &&
3201 "Any congruence class with a store should have a "
3202 "representative access");
3205 if (CC->getMemoryLeader())
3206 assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&
3207 "Representative MemoryAccess does not appear to be reverse "
3209 for (auto M : CC->memory())
3210 assert(MemoryAccessToClass.lookup(M) == CC &&
3211 "Memory member does not appear to be reverse mapped properly");
3214 // Anything equivalent in the MemoryAccess table should be in the same
3215 // congruence class.
3217 // Filter out the unreachable and trivially dead entries, because they may
3218 // never have been updated if the instructions were not processed.
3219 auto ReachableAccessPred =
3220 [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {
3221 bool Result = ReachableBlocks.count(Pair.first->getBlock());
3222 if (!Result || MSSA->isLiveOnEntryDef(Pair.first) ||
3223 MemoryToDFSNum(Pair.first) == 0)
3225 if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
3226 return !isInstructionTriviallyDead(MemDef->getMemoryInst());
3228 // We could have phi nodes which operands are all trivially dead,
3229 // so we don't process them.
3230 if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) {
3231 for (auto &U : MemPHI->incoming_values()) {
3232 if (auto *I = dyn_cast<Instruction>(&*U)) {
3233 if (!isInstructionTriviallyDead(I))
3243 auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);
3244 for (auto KV : Filtered) {
3245 if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
3246 auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());
3247 if (FirstMUD && SecondMUD) {
3248 SmallPtrSet<const MemoryAccess *, 8> VisitedMAS;
3249 assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||
3250 ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
3251 ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
3252 "The instructions for these memory operations should have "
3253 "been in the same congruence class or reachable through"
3254 "a single argument phi");
3256 } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
3257 // We can only sanely verify that MemoryDefs in the operand list all have
3259 auto ReachableOperandPred = [&](const Use &U) {
3260 return ReachableEdges.count(
3261 {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&
3265 // All arguments should in the same class, ignoring unreachable arguments
3266 auto FilteredPhiArgs =
3267 make_filter_range(FirstMP->operands(), ReachableOperandPred);
3268 SmallVector<const CongruenceClass *, 16> PhiOpClasses;
3269 std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
3270 std::back_inserter(PhiOpClasses), [&](const Use &U) {
3271 const MemoryDef *MD = cast<MemoryDef>(U);
3272 return ValueToClass.lookup(MD->getMemoryInst());
3274 assert(is_splat(PhiOpClasses) &&
3275 "All MemoryPhi arguments should be in the same class");
3281 // Verify that the sparse propagation we did actually found the maximal fixpoint
3282 // We do this by storing the value to class mapping, touching all instructions,
3283 // and redoing the iteration to see if anything changed.
3284 void NewGVN::verifyIterationSettled(Function &F) {
3286 LLVM_DEBUG(dbgs() << "Beginning iteration verification\n");
3287 if (DebugCounter::isCounterSet(VNCounter))
3288 DebugCounter::setCounterValue(VNCounter, StartingVNCounter);
3290 // Note that we have to store the actual classes, as we may change existing
3291 // classes during iteration. This is because our memory iteration propagation
3292 // is not perfect, and so may waste a little work. But it should generate
3293 // exactly the same congruence classes we have now, with different IDs.
3294 std::map<const Value *, CongruenceClass> BeforeIteration;
3296 for (auto &KV : ValueToClass) {
3297 if (auto *I = dyn_cast<Instruction>(KV.first))
3298 // Skip unused/dead instructions.
3299 if (InstrToDFSNum(I) == 0)
3301 BeforeIteration.insert({KV.first, *KV.second});
3304 TouchedInstructions.set();
3305 TouchedInstructions.reset(0);
3306 iterateTouchedInstructions();
3307 DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>>
3309 for (const auto &KV : ValueToClass) {
3310 if (auto *I = dyn_cast<Instruction>(KV.first))
3311 // Skip unused/dead instructions.
3312 if (InstrToDFSNum(I) == 0)
3314 // We could sink these uses, but i think this adds a bit of clarity here as
3315 // to what we are comparing.
3316 auto *BeforeCC = &BeforeIteration.find(KV.first)->second;
3317 auto *AfterCC = KV.second;
3318 // Note that the classes can't change at this point, so we memoize the set
3320 if (!EqualClasses.count({BeforeCC, AfterCC})) {
3321 assert(BeforeCC->isEquivalentTo(AfterCC) &&
3322 "Value number changed after main loop completed!");
3323 EqualClasses.insert({BeforeCC, AfterCC});
3329 // Verify that for each store expression in the expression to class mapping,
3330 // only the latest appears, and multiple ones do not appear.
3331 // Because loads do not use the stored value when doing equality with stores,
3332 // if we don't erase the old store expressions from the table, a load can find
3333 // a no-longer valid StoreExpression.
3334 void NewGVN::verifyStoreExpressions() const {
3336 // This is the only use of this, and it's not worth defining a complicated
3337 // densemapinfo hash/equality function for it.
3339 std::pair<const Value *,
3340 std::tuple<const Value *, const CongruenceClass *, Value *>>>
3342 for (const auto &KV : ExpressionToClass) {
3343 if (auto *SE = dyn_cast<StoreExpression>(KV.first)) {
3344 // Make sure a version that will conflict with loads is not already there
3345 auto Res = StoreExpressionSet.insert(
3346 {SE->getOperand(0), std::make_tuple(SE->getMemoryLeader(), KV.second,
3347 SE->getStoredValue())});
3348 bool Okay = Res.second;
3349 // It's okay to have the same expression already in there if it is
3350 // identical in nature.
3351 // This can happen when the leader of the stored value changes over time.
3353 Okay = (std::get<1>(Res.first->second) == KV.second) &&
3354 (lookupOperandLeader(std::get<2>(Res.first->second)) ==
3355 lookupOperandLeader(SE->getStoredValue()));
3356 assert(Okay && "Stored expression conflict exists in expression table");
3357 auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst());
3358 assert(ValueExpr && ValueExpr->equals(*SE) &&
3359 "StoreExpression in ExpressionToClass is not latest "
3360 "StoreExpression for value");
3366 // This is the main value numbering loop, it iterates over the initial touched
3367 // instruction set, propagating value numbers, marking things touched, etc,
3368 // until the set of touched instructions is completely empty.
3369 void NewGVN::iterateTouchedInstructions() {
3370 unsigned int Iterations = 0;
3371 // Figure out where touchedinstructions starts
3372 int FirstInstr = TouchedInstructions.find_first();
3373 // Nothing set, nothing to iterate, just return.
3374 if (FirstInstr == -1)
3376 const BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr));
3377 while (TouchedInstructions.any()) {
3379 // Walk through all the instructions in all the blocks in RPO.
3380 // TODO: As we hit a new block, we should push and pop equalities into a
3381 // table lookupOperandLeader can use, to catch things PredicateInfo
3382 // might miss, like edge-only equivalences.
3383 for (unsigned InstrNum : TouchedInstructions.set_bits()) {
3385 // This instruction was found to be dead. We don't bother looking
3387 if (InstrNum == 0) {
3388 TouchedInstructions.reset(InstrNum);
3392 Value *V = InstrFromDFSNum(InstrNum);
3393 const BasicBlock *CurrBlock = getBlockForValue(V);
3395 // If we hit a new block, do reachability processing.
3396 if (CurrBlock != LastBlock) {
3397 LastBlock = CurrBlock;
3398 bool BlockReachable = ReachableBlocks.count(CurrBlock);
3399 const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);
3401 // If it's not reachable, erase any touched instructions and move on.
3402 if (!BlockReachable) {
3403 TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);
3404 LLVM_DEBUG(dbgs() << "Skipping instructions in block "
3405 << getBlockName(CurrBlock)
3406 << " because it is unreachable\n");
3409 updateProcessedCount(CurrBlock);
3411 // Reset after processing (because we may mark ourselves as touched when
3412 // we propagate equalities).
3413 TouchedInstructions.reset(InstrNum);
3415 if (auto *MP = dyn_cast<MemoryPhi>(V)) {
3416 LLVM_DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
3417 valueNumberMemoryPhi(MP);
3418 } else if (auto *I = dyn_cast<Instruction>(V)) {
3419 valueNumberInstruction(I);
3421 llvm_unreachable("Should have been a MemoryPhi or Instruction");
3423 updateProcessedCount(V);
3426 NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);
3429 // This is the main transformation entry point.
3430 bool NewGVN::runGVN() {
3431 if (DebugCounter::isCounterSet(VNCounter))
3432 StartingVNCounter = DebugCounter::getCounterValue(VNCounter);
3433 bool Changed = false;
3434 NumFuncArgs = F.arg_size();
3435 MSSAWalker = MSSA->getWalker();
3436 SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();
3438 // Count number of instructions for sizing of hash tables, and come
3439 // up with a global dfs numbering for instructions.
3440 unsigned ICount = 1;
3441 // Add an empty instruction to account for the fact that we start at 1
3442 DFSToInstr.emplace_back(nullptr);
3443 // Note: We want ideal RPO traversal of the blocks, which is not quite the
3444 // same as dominator tree order, particularly with regard whether backedges
3445 // get visited first or second, given a block with multiple successors.
3446 // If we visit in the wrong order, we will end up performing N times as many
3448 // The dominator tree does guarantee that, for a given dom tree node, it's
3449 // parent must occur before it in the RPO ordering. Thus, we only need to sort
3451 ReversePostOrderTraversal<Function *> RPOT(&F);
3452 unsigned Counter = 0;
3453 for (auto &B : RPOT) {
3454 auto *Node = DT->getNode(B);
3455 assert(Node && "RPO and Dominator tree should have same reachability");
3456 RPOOrdering[Node] = ++Counter;
3458 // Sort dominator tree children arrays into RPO.
3459 for (auto &B : RPOT) {
3460 auto *Node = DT->getNode(B);
3461 if (Node->getChildren().size() > 1)
3462 llvm::sort(Node->begin(), Node->end(),
3463 [&](const DomTreeNode *A, const DomTreeNode *B) {
3464 return RPOOrdering[A] < RPOOrdering[B];
3468 // Now a standard depth first ordering of the domtree is equivalent to RPO.
3469 for (auto DTN : depth_first(DT->getRootNode())) {
3470 BasicBlock *B = DTN->getBlock();
3471 const auto &BlockRange = assignDFSNumbers(B, ICount);
3472 BlockInstRange.insert({B, BlockRange});
3473 ICount += BlockRange.second - BlockRange.first;
3475 initializeCongruenceClasses(F);
3477 TouchedInstructions.resize(ICount);
3478 // Ensure we don't end up resizing the expressionToClass map, as
3479 // that can be quite expensive. At most, we have one expression per
3481 ExpressionToClass.reserve(ICount);
3483 // Initialize the touched instructions to include the entry block.
3484 const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());
3485 TouchedInstructions.set(InstRange.first, InstRange.second);
3486 LLVM_DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())
3487 << " marked reachable\n");
3488 ReachableBlocks.insert(&F.getEntryBlock());
3490 iterateTouchedInstructions();
3491 verifyMemoryCongruency();
3492 verifyIterationSettled(F);
3493 verifyStoreExpressions();
3495 Changed |= eliminateInstructions(F);
3497 // Delete all instructions marked for deletion.
3498 for (Instruction *ToErase : InstructionsToErase) {
3499 if (!ToErase->use_empty())
3500 ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType()));
3502 assert(ToErase->getParent() &&
3503 "BB containing ToErase deleted unexpectedly!");
3504 ToErase->eraseFromParent();
3506 Changed |= !InstructionsToErase.empty();
3508 // Delete all unreachable blocks.
3509 auto UnreachableBlockPred = [&](const BasicBlock &BB) {
3510 return !ReachableBlocks.count(&BB);
3513 for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {
3514 LLVM_DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
3515 << " is unreachable\n");
3516 deleteInstructionsInBlock(&BB);
3524 struct NewGVN::ValueDFS {
3529 // Only one of Def and U will be set.
3530 // The bool in the Def tells us whether the Def is the stored value of a
3532 PointerIntPair<Value *, 1, bool> Def;
3535 bool operator<(const ValueDFS &Other) const {
3536 // It's not enough that any given field be less than - we have sets
3537 // of fields that need to be evaluated together to give a proper ordering.
3538 // For example, if you have;
3543 // We want the second to be less than the first, but if we just go field
3544 // by field, we will get to Val 0 < Val 50 and say the first is less than
3545 // the second. We only want it to be less than if the DFS orders are equal.
3547 // Each LLVM instruction only produces one value, and thus the lowest-level
3548 // differentiator that really matters for the stack (and what we use as as a
3549 // replacement) is the local dfs number.
3550 // Everything else in the structure is instruction level, and only affects
3551 // the order in which we will replace operands of a given instruction.
3553 // For a given instruction (IE things with equal dfsin, dfsout, localnum),
3554 // the order of replacement of uses does not matter.
3558 // When you hit b, you will have two valuedfs with the same dfsin, out, and
3560 // The .val will be the same as well.
3561 // The .u's will be different.
3562 // You will replace both, and it does not matter what order you replace them
3563 // in (IE whether you replace operand 2, then operand 1, or operand 1, then
3565 // Similarly for the case of same dfsin, dfsout, localnum, but different
3570 // in c, we will a valuedfs for a, and one for b,with everything the same
3572 // It does not matter what order we replace these operands in.
3573 // You will always end up with the same IR, and this is guaranteed.
3574 return std::tie(DFSIn, DFSOut, LocalNum, Def, U) <
3575 std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def,
3580 // This function converts the set of members for a congruence class from values,
3581 // to sets of defs and uses with associated DFS info. The total number of
3582 // reachable uses for each value is stored in UseCount, and instructions that
3584 // dead (have no non-dead uses) are stored in ProbablyDead.
3585 void NewGVN::convertClassToDFSOrdered(
3586 const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,
3587 DenseMap<const Value *, unsigned int> &UseCounts,
3588 SmallPtrSetImpl<Instruction *> &ProbablyDead) const {
3589 for (auto D : Dense) {
3590 // First add the value.
3591 BasicBlock *BB = getBlockForValue(D);
3592 // Constants are handled prior to ever calling this function, so
3593 // we should only be left with instructions as members.
3594 assert(BB && "Should have figured out a basic block for value");
3596 DomTreeNode *DomNode = DT->getNode(BB);
3597 VDDef.DFSIn = DomNode->getDFSNumIn();
3598 VDDef.DFSOut = DomNode->getDFSNumOut();
3599 // If it's a store, use the leader of the value operand, if it's always
3600 // available, or the value operand. TODO: We could do dominance checks to
3601 // find a dominating leader, but not worth it ATM.
3602 if (auto *SI = dyn_cast<StoreInst>(D)) {
3603 auto Leader = lookupOperandLeader(SI->getValueOperand());
3604 if (alwaysAvailable(Leader)) {
3605 VDDef.Def.setPointer(Leader);
3607 VDDef.Def.setPointer(SI->getValueOperand());
3608 VDDef.Def.setInt(true);
3611 VDDef.Def.setPointer(D);
3613 assert(isa<Instruction>(D) &&
3614 "The dense set member should always be an instruction");
3615 Instruction *Def = cast<Instruction>(D);
3616 VDDef.LocalNum = InstrToDFSNum(D);
3617 DFSOrderedSet.push_back(VDDef);
3618 // If there is a phi node equivalent, add it
3619 if (auto *PN = RealToTemp.lookup(Def)) {
3621 dyn_cast_or_null<PHIExpression>(ValueToExpression.lookup(Def));
3623 VDDef.Def.setInt(false);
3624 VDDef.Def.setPointer(PN);
3626 DFSOrderedSet.push_back(VDDef);
3630 unsigned int UseCount = 0;
3631 // Now add the uses.
3632 for (auto &U : Def->uses()) {
3633 if (auto *I = dyn_cast<Instruction>(U.getUser())) {
3634 // Don't try to replace into dead uses
3635 if (InstructionsToErase.count(I))
3638 // Put the phi node uses in the incoming block.
3640 if (auto *P = dyn_cast<PHINode>(I)) {
3641 IBlock = P->getIncomingBlock(U);
3642 // Make phi node users appear last in the incoming block
3644 VDUse.LocalNum = InstrDFS.size() + 1;
3646 IBlock = getBlockForValue(I);
3647 VDUse.LocalNum = InstrToDFSNum(I);
3650 // Skip uses in unreachable blocks, as we're going
3652 if (ReachableBlocks.count(IBlock) == 0)
3655 DomTreeNode *DomNode = DT->getNode(IBlock);
3656 VDUse.DFSIn = DomNode->getDFSNumIn();
3657 VDUse.DFSOut = DomNode->getDFSNumOut();
3660 DFSOrderedSet.emplace_back(VDUse);
3664 // If there are no uses, it's probably dead (but it may have side-effects,
3665 // so not definitely dead. Otherwise, store the number of uses so we can
3666 // track if it becomes dead later).
3668 ProbablyDead.insert(Def);
3670 UseCounts[Def] = UseCount;
3674 // This function converts the set of members for a congruence class from values,
3675 // to the set of defs for loads and stores, with associated DFS info.
3676 void NewGVN::convertClassToLoadsAndStores(
3677 const CongruenceClass &Dense,
3678 SmallVectorImpl<ValueDFS> &LoadsAndStores) const {
3679 for (auto D : Dense) {
3680 if (!isa<LoadInst>(D) && !isa<StoreInst>(D))
3683 BasicBlock *BB = getBlockForValue(D);
3685 DomTreeNode *DomNode = DT->getNode(BB);
3686 VD.DFSIn = DomNode->getDFSNumIn();
3687 VD.DFSOut = DomNode->getDFSNumOut();
3688 VD.Def.setPointer(D);
3690 // If it's an instruction, use the real local dfs number.
3691 if (auto *I = dyn_cast<Instruction>(D))
3692 VD.LocalNum = InstrToDFSNum(I);
3694 llvm_unreachable("Should have been an instruction");
3696 LoadsAndStores.emplace_back(VD);
3700 static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
3701 patchReplacementInstruction(I, Repl);
3702 I->replaceAllUsesWith(Repl);
3705 void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
3706 LLVM_DEBUG(dbgs() << " BasicBlock Dead:" << *BB);
3707 ++NumGVNBlocksDeleted;
3709 // Delete the instructions backwards, as it has a reduced likelihood of having
3710 // to update as many def-use and use-def chains. Start after the terminator.
3711 auto StartPoint = BB->rbegin();
3713 // Note that we explicitly recalculate BB->rend() on each iteration,
3714 // as it may change when we remove the first instruction.
3715 for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
3716 Instruction &Inst = *I++;
3717 if (!Inst.use_empty())
3718 Inst.replaceAllUsesWith(UndefValue::get(Inst.getType()));
3719 if (isa<LandingPadInst>(Inst))
3722 Inst.eraseFromParent();
3723 ++NumGVNInstrDeleted;
3725 // Now insert something that simplifycfg will turn into an unreachable.
3726 Type *Int8Ty = Type::getInt8Ty(BB->getContext());
3727 new StoreInst(UndefValue::get(Int8Ty),
3728 Constant::getNullValue(Int8Ty->getPointerTo()),
3729 BB->getTerminator());
3732 void NewGVN::markInstructionForDeletion(Instruction *I) {
3733 LLVM_DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
3734 InstructionsToErase.insert(I);
3737 void NewGVN::replaceInstruction(Instruction *I, Value *V) {
3738 LLVM_DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
3739 patchAndReplaceAllUsesWith(I, V);
3740 // We save the actual erasing to avoid invalidating memory
3741 // dependencies until we are done with everything.
3742 markInstructionForDeletion(I);
3747 // This is a stack that contains both the value and dfs info of where
3748 // that value is valid.
3749 class ValueDFSStack {
3751 Value *back() const { return ValueStack.back(); }
3752 std::pair<int, int> dfs_back() const { return DFSStack.back(); }
3754 void push_back(Value *V, int DFSIn, int DFSOut) {
3755 ValueStack.emplace_back(V);
3756 DFSStack.emplace_back(DFSIn, DFSOut);
3759 bool empty() const { return DFSStack.empty(); }
3761 bool isInScope(int DFSIn, int DFSOut) const {
3764 return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
3767 void popUntilDFSScope(int DFSIn, int DFSOut) {
3769 // These two should always be in sync at this point.
3770 assert(ValueStack.size() == DFSStack.size() &&
3771 "Mismatch between ValueStack and DFSStack");
3773 !DFSStack.empty() &&
3774 !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {
3775 DFSStack.pop_back();
3776 ValueStack.pop_back();
3781 SmallVector<Value *, 8> ValueStack;
3782 SmallVector<std::pair<int, int>, 8> DFSStack;
3785 } // end anonymous namespace
3787 // Given an expression, get the congruence class for it.
3788 CongruenceClass *NewGVN::getClassForExpression(const Expression *E) const {
3789 if (auto *VE = dyn_cast<VariableExpression>(E))
3790 return ValueToClass.lookup(VE->getVariableValue());
3791 else if (isa<DeadExpression>(E))
3793 return ExpressionToClass.lookup(E);
3796 // Given a value and a basic block we are trying to see if it is available in,
3797 // see if the value has a leader available in that block.
3798 Value *NewGVN::findPHIOfOpsLeader(const Expression *E,
3799 const Instruction *OrigInst,
3800 const BasicBlock *BB) const {
3801 // It would already be constant if we could make it constant
3802 if (auto *CE = dyn_cast<ConstantExpression>(E))
3803 return CE->getConstantValue();
3804 if (auto *VE = dyn_cast<VariableExpression>(E)) {
3805 auto *V = VE->getVariableValue();
3806 if (alwaysAvailable(V) || DT->dominates(getBlockForValue(V), BB))
3807 return VE->getVariableValue();
3810 auto *CC = getClassForExpression(E);
3813 if (alwaysAvailable(CC->getLeader()))
3814 return CC->getLeader();
3816 for (auto Member : *CC) {
3817 auto *MemberInst = dyn_cast<Instruction>(Member);
3818 if (MemberInst == OrigInst)
3820 // Anything that isn't an instruction is always available.
3823 if (DT->dominates(getBlockForValue(MemberInst), BB))
3829 bool NewGVN::eliminateInstructions(Function &F) {
3830 // This is a non-standard eliminator. The normal way to eliminate is
3831 // to walk the dominator tree in order, keeping track of available
3832 // values, and eliminating them. However, this is mildly
3833 // pointless. It requires doing lookups on every instruction,
3834 // regardless of whether we will ever eliminate it. For
3835 // instructions part of most singleton congruence classes, we know we
3836 // will never eliminate them.
3838 // Instead, this eliminator looks at the congruence classes directly, sorts
3839 // them into a DFS ordering of the dominator tree, and then we just
3840 // perform elimination straight on the sets by walking the congruence
3841 // class member uses in order, and eliminate the ones dominated by the
3842 // last member. This is worst case O(E log E) where E = number of
3843 // instructions in a single congruence class. In theory, this is all
3844 // instructions. In practice, it is much faster, as most instructions are
3845 // either in singleton congruence classes or can't possibly be eliminated
3846 // anyway (if there are no overlapping DFS ranges in class).
3847 // When we find something not dominated, it becomes the new leader
3848 // for elimination purposes.
3849 // TODO: If we wanted to be faster, We could remove any members with no
3850 // overlapping ranges while sorting, as we will never eliminate anything
3851 // with those members, as they don't dominate anything else in our set.
3853 bool AnythingReplaced = false;
3855 // Since we are going to walk the domtree anyway, and we can't guarantee the
3856 // DFS numbers are updated, we compute some ourselves.
3857 DT->updateDFSNumbers();
3859 // Go through all of our phi nodes, and kill the arguments associated with
3860 // unreachable edges.
3861 auto ReplaceUnreachablePHIArgs = [&](PHINode *PHI, BasicBlock *BB) {
3862 for (auto &Operand : PHI->incoming_values())
3863 if (!ReachableEdges.count({PHI->getIncomingBlock(Operand), BB})) {
3864 LLVM_DEBUG(dbgs() << "Replacing incoming value of " << PHI
3866 << getBlockName(PHI->getIncomingBlock(Operand))
3867 << " with undef due to it being unreachable\n");
3868 Operand.set(UndefValue::get(PHI->getType()));
3871 // Replace unreachable phi arguments.
3872 // At this point, RevisitOnReachabilityChange only contains:
3875 // 2. Temporaries that will convert to PHIs
3876 // 3. Operations that are affected by an unreachable edge but do not fit into
3878 // So it is a slight overshoot of what we want. We could make it exact by
3879 // using two SparseBitVectors per block.
3880 DenseMap<const BasicBlock *, unsigned> ReachablePredCount;
3881 for (auto &KV : ReachableEdges)
3882 ReachablePredCount[KV.getEnd()]++;
3883 for (auto &BBPair : RevisitOnReachabilityChange) {
3884 for (auto InstNum : BBPair.second) {
3885 auto *Inst = InstrFromDFSNum(InstNum);
3886 auto *PHI = dyn_cast<PHINode>(Inst);
3887 PHI = PHI ? PHI : dyn_cast_or_null<PHINode>(RealToTemp.lookup(Inst));
3890 auto *BB = BBPair.first;
3891 if (ReachablePredCount.lookup(BB) != PHI->getNumIncomingValues())
3892 ReplaceUnreachablePHIArgs(PHI, BB);
3896 // Map to store the use counts
3897 DenseMap<const Value *, unsigned int> UseCounts;
3898 for (auto *CC : reverse(CongruenceClasses)) {
3899 LLVM_DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID()
3901 // Track the equivalent store info so we can decide whether to try
3902 // dead store elimination.
3903 SmallVector<ValueDFS, 8> PossibleDeadStores;
3904 SmallPtrSet<Instruction *, 8> ProbablyDead;
3905 if (CC->isDead() || CC->empty())
3907 // Everything still in the TOP class is unreachable or dead.
3908 if (CC == TOPClass) {
3909 for (auto M : *CC) {
3910 auto *VTE = ValueToExpression.lookup(M);
3911 if (VTE && isa<DeadExpression>(VTE))
3912 markInstructionForDeletion(cast<Instruction>(M));
3913 assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||
3914 InstructionsToErase.count(cast<Instruction>(M))) &&
3915 "Everything in TOP should be unreachable or dead at this "
3921 assert(CC->getLeader() && "We should have had a leader");
3922 // If this is a leader that is always available, and it's a
3923 // constant or has no equivalences, just replace everything with
3924 // it. We then update the congruence class with whatever members
3927 CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
3928 if (alwaysAvailable(Leader)) {
3929 CongruenceClass::MemberSet MembersLeft;
3930 for (auto M : *CC) {
3932 // Void things have no uses we can replace.
3933 if (Member == Leader || !isa<Instruction>(Member) ||
3934 Member->getType()->isVoidTy()) {
3935 MembersLeft.insert(Member);
3938 LLVM_DEBUG(dbgs() << "Found replacement " << *(Leader) << " for "
3939 << *Member << "\n");
3940 auto *I = cast<Instruction>(Member);
3941 assert(Leader != I && "About to accidentally remove our leader");
3942 replaceInstruction(I, Leader);
3943 AnythingReplaced = true;
3945 CC->swap(MembersLeft);
3947 // If this is a singleton, we can skip it.
3948 if (CC->size() != 1 || RealToTemp.count(Leader)) {
3949 // This is a stack because equality replacement/etc may place
3950 // constants in the middle of the member list, and we want to use
3951 // those constant values in preference to the current leader, over
3952 // the scope of those constants.
3953 ValueDFSStack EliminationStack;
3955 // Convert the members to DFS ordered sets and then merge them.
3956 SmallVector<ValueDFS, 8> DFSOrderedSet;
3957 convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead);
3959 // Sort the whole thing.
3960 llvm::sort(DFSOrderedSet);
3961 for (auto &VD : DFSOrderedSet) {
3962 int MemberDFSIn = VD.DFSIn;
3963 int MemberDFSOut = VD.DFSOut;
3964 Value *Def = VD.Def.getPointer();
3965 bool FromStore = VD.Def.getInt();
3967 // We ignore void things because we can't get a value from them.
3968 if (Def && Def->getType()->isVoidTy())
3970 auto *DefInst = dyn_cast_or_null<Instruction>(Def);
3971 if (DefInst && AllTempInstructions.count(DefInst)) {
3972 auto *PN = cast<PHINode>(DefInst);
3974 // If this is a value phi and that's the expression we used, insert
3975 // it into the program
3976 // remove from temp instruction list.
3977 AllTempInstructions.erase(PN);
3978 auto *DefBlock = getBlockForValue(Def);
3979 LLVM_DEBUG(dbgs() << "Inserting fully real phi of ops" << *Def
3981 << getBlockName(getBlockForValue(Def)) << "\n");
3982 PN->insertBefore(&DefBlock->front());
3984 NumGVNPHIOfOpsEliminations++;
3987 if (EliminationStack.empty()) {
3988 LLVM_DEBUG(dbgs() << "Elimination Stack is empty\n");
3990 LLVM_DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
3991 << EliminationStack.dfs_back().first << ","
3992 << EliminationStack.dfs_back().second << ")\n");
3995 LLVM_DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
3996 << MemberDFSOut << ")\n");
3997 // First, we see if we are out of scope or empty. If so,
3998 // and there equivalences, we try to replace the top of
3999 // stack with equivalences (if it's on the stack, it must
4000 // not have been eliminated yet).
4001 // Then we synchronize to our current scope, by
4002 // popping until we are back within a DFS scope that
4003 // dominates the current member.
4004 // Then, what happens depends on a few factors
4005 // If the stack is now empty, we need to push
4006 // If we have a constant or a local equivalence we want to
4007 // start using, we also push.
4008 // Otherwise, we walk along, processing members who are
4009 // dominated by this scope, and eliminate them.
4010 bool ShouldPush = Def && EliminationStack.empty();
4012 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);
4014 if (OutOfScope || ShouldPush) {
4015 // Sync to our current scope.
4016 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
4017 bool ShouldPush = Def && EliminationStack.empty();
4019 EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut);
4023 // Skip the Def's, we only want to eliminate on their uses. But mark
4024 // dominated defs as dead.
4026 // For anything in this case, what and how we value number
4027 // guarantees that any side-effets that would have occurred (ie
4028 // throwing, etc) can be proven to either still occur (because it's
4029 // dominated by something that has the same side-effects), or never
4030 // occur. Otherwise, we would not have been able to prove it value
4031 // equivalent to something else. For these things, we can just mark
4032 // it all dead. Note that this is different from the "ProbablyDead"
4033 // set, which may not be dominated by anything, and thus, are only
4034 // easy to prove dead if they are also side-effect free. Note that
4035 // because stores are put in terms of the stored value, we skip
4036 // stored values here. If the stored value is really dead, it will
4037 // still be marked for deletion when we process it in its own class.
4038 if (!EliminationStack.empty() && Def != EliminationStack.back() &&
4039 isa<Instruction>(Def) && !FromStore)
4040 markInstructionForDeletion(cast<Instruction>(Def));
4043 // At this point, we know it is a Use we are trying to possibly
4046 assert(isa<Instruction>(U->get()) &&
4047 "Current def should have been an instruction");
4048 assert(isa<Instruction>(U->getUser()) &&
4049 "Current user should have been an instruction");
4051 // If the thing we are replacing into is already marked to be dead,
4052 // this use is dead. Note that this is true regardless of whether
4053 // we have anything dominating the use or not. We do this here
4054 // because we are already walking all the uses anyway.
4055 Instruction *InstUse = cast<Instruction>(U->getUser());
4056 if (InstructionsToErase.count(InstUse)) {
4057 auto &UseCount = UseCounts[U->get()];
4058 if (--UseCount == 0) {
4059 ProbablyDead.insert(cast<Instruction>(U->get()));
4063 // If we get to this point, and the stack is empty we must have a use
4064 // with nothing we can use to eliminate this use, so just skip it.
4065 if (EliminationStack.empty())
4068 Value *DominatingLeader = EliminationStack.back();
4070 auto *II = dyn_cast<IntrinsicInst>(DominatingLeader);
4071 bool isSSACopy = II && II->getIntrinsicID() == Intrinsic::ssa_copy;
4073 DominatingLeader = II->getOperand(0);
4075 // Don't replace our existing users with ourselves.
4076 if (U->get() == DominatingLeader)
4079 << "Found replacement " << *DominatingLeader << " for "
4080 << *U->get() << " in " << *(U->getUser()) << "\n");
4082 // If we replaced something in an instruction, handle the patching of
4083 // metadata. Skip this if we are replacing predicateinfo with its
4084 // original operand, as we already know we can just drop it.
4085 auto *ReplacedInst = cast<Instruction>(U->get());
4086 auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst);
4087 if (!PI || DominatingLeader != PI->OriginalOp)
4088 patchReplacementInstruction(ReplacedInst, DominatingLeader);
4089 U->set(DominatingLeader);
4090 // This is now a use of the dominating leader, which means if the
4091 // dominating leader was dead, it's now live!
4092 auto &LeaderUseCount = UseCounts[DominatingLeader];
4093 // It's about to be alive again.
4094 if (LeaderUseCount == 0 && isa<Instruction>(DominatingLeader))
4095 ProbablyDead.erase(cast<Instruction>(DominatingLeader));
4096 // For copy instructions, we use their operand as a leader,
4097 // which means we remove a user of the copy and it may become dead.
4099 unsigned &IIUseCount = UseCounts[II];
4100 if (--IIUseCount == 0)
4101 ProbablyDead.insert(II);
4104 AnythingReplaced = true;
4109 // At this point, anything still in the ProbablyDead set is actually dead if
4110 // would be trivially dead.
4111 for (auto *I : ProbablyDead)
4112 if (wouldInstructionBeTriviallyDead(I))
4113 markInstructionForDeletion(I);
4115 // Cleanup the congruence class.
4116 CongruenceClass::MemberSet MembersLeft;
4117 for (auto *Member : *CC)
4118 if (!isa<Instruction>(Member) ||
4119 !InstructionsToErase.count(cast<Instruction>(Member)))
4120 MembersLeft.insert(Member);
4121 CC->swap(MembersLeft);
4123 // If we have possible dead stores to look at, try to eliminate them.
4124 if (CC->getStoreCount() > 0) {
4125 convertClassToLoadsAndStores(*CC, PossibleDeadStores);
4126 llvm::sort(PossibleDeadStores);
4127 ValueDFSStack EliminationStack;
4128 for (auto &VD : PossibleDeadStores) {
4129 int MemberDFSIn = VD.DFSIn;
4130 int MemberDFSOut = VD.DFSOut;
4131 Instruction *Member = cast<Instruction>(VD.Def.getPointer());
4132 if (EliminationStack.empty() ||
4133 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)) {
4134 // Sync to our current scope.
4135 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
4136 if (EliminationStack.empty()) {
4137 EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);
4141 // We already did load elimination, so nothing to do here.
4142 if (isa<LoadInst>(Member))
4144 assert(!EliminationStack.empty());
4145 Instruction *Leader = cast<Instruction>(EliminationStack.back());
4147 assert(DT->dominates(Leader->getParent(), Member->getParent()));
4148 // Member is dominater by Leader, and thus dead
4149 LLVM_DEBUG(dbgs() << "Marking dead store " << *Member
4150 << " that is dominated by " << *Leader << "\n");
4151 markInstructionForDeletion(Member);
4157 return AnythingReplaced;
4160 // This function provides global ranking of operations so that we can place them
4161 // in a canonical order. Note that rank alone is not necessarily enough for a
4162 // complete ordering, as constants all have the same rank. However, generally,
4163 // we will simplify an operation with all constants so that it doesn't matter
4164 // what order they appear in.
4165 unsigned int NewGVN::getRank(const Value *V) const {
4166 // Prefer constants to undef to anything else
4167 // Undef is a constant, have to check it first.
4168 // Prefer smaller constants to constantexprs
4169 if (isa<ConstantExpr>(V))
4171 if (isa<UndefValue>(V))
4173 if (isa<Constant>(V))
4175 else if (auto *A = dyn_cast<Argument>(V))
4176 return 3 + A->getArgNo();
4178 // Need to shift the instruction DFS by number of arguments + 3 to account for
4179 // the constant and argument ranking above.
4180 unsigned Result = InstrToDFSNum(V);
4182 return 4 + NumFuncArgs + Result;
4183 // Unreachable or something else, just return a really large number.
4187 // This is a function that says whether two commutative operations should
4188 // have their order swapped when canonicalizing.
4189 bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
4190 // Because we only care about a total ordering, and don't rewrite expressions
4191 // in this order, we order by rank, which will give a strict weak ordering to
4192 // everything but constants, and then we order by pointer address.
4193 return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);
4198 class NewGVNLegacyPass : public FunctionPass {
4200 // Pass identification, replacement for typeid.
4203 NewGVNLegacyPass() : FunctionPass(ID) {
4204 initializeNewGVNLegacyPassPass(*PassRegistry::getPassRegistry());
4207 bool runOnFunction(Function &F) override;
4210 void getAnalysisUsage(AnalysisUsage &AU) const override {
4211 AU.addRequired<AssumptionCacheTracker>();
4212 AU.addRequired<DominatorTreeWrapperPass>();
4213 AU.addRequired<TargetLibraryInfoWrapperPass>();
4214 AU.addRequired<MemorySSAWrapperPass>();
4215 AU.addRequired<AAResultsWrapperPass>();
4216 AU.addPreserved<DominatorTreeWrapperPass>();
4217 AU.addPreserved<GlobalsAAWrapperPass>();
4221 } // end anonymous namespace
4223 bool NewGVNLegacyPass::runOnFunction(Function &F) {
4224 if (skipFunction(F))
4226 return NewGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
4227 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
4228 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
4229 &getAnalysis<AAResultsWrapperPass>().getAAResults(),
4230 &getAnalysis<MemorySSAWrapperPass>().getMSSA(),
4231 F.getParent()->getDataLayout())
4235 char NewGVNLegacyPass::ID = 0;
4237 INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering",
4239 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
4240 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
4241 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
4242 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
4243 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
4244 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
4245 INITIALIZE_PASS_END(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false,
4248 // createGVNPass - The public interface to this file.
4249 FunctionPass *llvm::createNewGVNPass() { return new NewGVNLegacyPass(); }
4251 PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
4252 // Apparently the order in which we get these results matter for
4253 // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
4254 // the same order here, just in case.
4255 auto &AC = AM.getResult<AssumptionAnalysis>(F);
4256 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
4257 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
4258 auto &AA = AM.getResult<AAManager>(F);
4259 auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
4261 NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getParent()->getDataLayout())
4264 return PreservedAnalyses::all();
4265 PreservedAnalyses PA;
4266 PA.preserve<DominatorTreeAnalysis>();
4267 PA.preserve<GlobalsAA>();