1 //===---- NewGVN.cpp - Global Value Numbering Pass --------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// This file implements the new LLVM's Global Value Numbering pass.
11 /// GVN partitions values computed by a function into congruence classes.
12 /// Values ending up in the same congruence class are guaranteed to be the same
13 /// for every execution of the program. In that respect, congruency is a
14 /// compile-time approximation of equivalence of values at runtime.
15 /// The algorithm implemented here uses a sparse formulation and it's based
16 /// on the ideas described in the paper:
17 /// "A Sparse Algorithm for Predicated Global Value Numbering" from
20 /// A brief overview of the algorithm: The algorithm is essentially the same as
21 /// the standard RPO value numbering algorithm (a good reference is the paper
22 /// "SCC based value numbering" by L. Taylor Simpson) with one major difference:
23 /// The RPO algorithm proceeds, on every iteration, to process every reachable
24 /// block and every instruction in that block. This is because the standard RPO
25 /// algorithm does not track what things have the same value number, it only
26 /// tracks what the value number of a given operation is (the mapping is
27 /// operation -> value number). Thus, when a value number of an operation
28 /// changes, it must reprocess everything to ensure all uses of a value number
29 /// get updated properly. In constrast, the sparse algorithm we use *also*
30 /// tracks what operations have a given value number (IE it also tracks the
31 /// reverse mapping from value number -> operations with that value number), so
32 /// that it only needs to reprocess the instructions that are affected when
33 /// something's value number changes. The rest of the algorithm is devoted to
34 /// performing symbolic evaluation, forward propagation, and simplification of
35 /// operations based on the value numbers deduced so far.
37 /// We also do not perform elimination by using any published algorithm. All
38 /// published algorithms are O(Instructions). Instead, we use a technique that
39 /// is O(number of operations with the same value number), enabling us to skip
40 /// trying to eliminate things that have unique value numbers.
41 //===----------------------------------------------------------------------===//
43 #include "llvm/Transforms/Scalar/NewGVN.h"
44 #include "llvm/ADT/BitVector.h"
45 #include "llvm/ADT/DenseMap.h"
46 #include "llvm/ADT/DenseSet.h"
47 #include "llvm/ADT/DepthFirstIterator.h"
48 #include "llvm/ADT/Hashing.h"
49 #include "llvm/ADT/MapVector.h"
50 #include "llvm/ADT/PostOrderIterator.h"
51 #include "llvm/ADT/STLExtras.h"
52 #include "llvm/ADT/SmallPtrSet.h"
53 #include "llvm/ADT/SmallSet.h"
54 #include "llvm/ADT/SparseBitVector.h"
55 #include "llvm/ADT/Statistic.h"
56 #include "llvm/ADT/TinyPtrVector.h"
57 #include "llvm/Analysis/AliasAnalysis.h"
58 #include "llvm/Analysis/AssumptionCache.h"
59 #include "llvm/Analysis/CFG.h"
60 #include "llvm/Analysis/CFGPrinter.h"
61 #include "llvm/Analysis/ConstantFolding.h"
62 #include "llvm/Analysis/GlobalsModRef.h"
63 #include "llvm/Analysis/InstructionSimplify.h"
64 #include "llvm/Analysis/MemoryBuiltins.h"
65 #include "llvm/Analysis/MemoryLocation.h"
66 #include "llvm/Analysis/MemorySSA.h"
67 #include "llvm/Analysis/TargetLibraryInfo.h"
68 #include "llvm/IR/DataLayout.h"
69 #include "llvm/IR/Dominators.h"
70 #include "llvm/IR/GlobalVariable.h"
71 #include "llvm/IR/IRBuilder.h"
72 #include "llvm/IR/IntrinsicInst.h"
73 #include "llvm/IR/LLVMContext.h"
74 #include "llvm/IR/Metadata.h"
75 #include "llvm/IR/PatternMatch.h"
76 #include "llvm/IR/Type.h"
77 #include "llvm/Support/Allocator.h"
78 #include "llvm/Support/CommandLine.h"
79 #include "llvm/Support/Debug.h"
80 #include "llvm/Support/DebugCounter.h"
81 #include "llvm/Transforms/Scalar.h"
82 #include "llvm/Transforms/Scalar/GVNExpression.h"
83 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
84 #include "llvm/Transforms/Utils/Local.h"
85 #include "llvm/Transforms/Utils/PredicateInfo.h"
86 #include "llvm/Transforms/Utils/VNCoercion.h"
88 #include <unordered_map>
92 using namespace PatternMatch;
93 using namespace llvm::GVNExpression;
94 using namespace llvm::VNCoercion;
95 #define DEBUG_TYPE "newgvn"
97 STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
98 STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
99 STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
100 STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
101 STATISTIC(NumGVNMaxIterations,
102 "Maximum Number of iterations it took to converge GVN");
103 STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
104 STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
105 STATISTIC(NumGVNAvoidedSortedLeaderChanges,
106 "Number of avoided sorted leader changes");
107 STATISTIC(NumGVNNotMostDominatingLeader,
108 "Number of times a member dominated it's new classes' leader");
109 STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");
110 DEBUG_COUNTER(VNCounter, "newgvn-vn",
111 "Controls which instructions are value numbered")
113 // Currently store defining access refinement is too slow due to basicaa being
114 // egregiously slow. This flag lets us keep it working while we work on this
116 static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
117 cl::init(false), cl::Hidden);
119 //===----------------------------------------------------------------------===//
121 //===----------------------------------------------------------------------===//
125 namespace GVNExpression {
126 Expression::~Expression() = default;
127 BasicExpression::~BasicExpression() = default;
128 CallExpression::~CallExpression() = default;
129 LoadExpression::~LoadExpression() = default;
130 StoreExpression::~StoreExpression() = default;
131 AggregateValueExpression::~AggregateValueExpression() = default;
132 PHIExpression::~PHIExpression() = default;
136 // Tarjan's SCC finding algorithm with Nuutila's improvements
137 // SCCIterator is actually fairly complex for the simple thing we want.
138 // It also wants to hand us SCC's that are unrelated to the phi node we ask
139 // about, and have us process them there or risk redoing work.
140 // Graph traits over a filter iterator also doesn't work that well here.
141 // This SCC finder is specialized to walk use-def chains, and only follows instructions,
142 // not generic values (arguments, etc).
145 TarjanSCC() : Components(1) {}
147 void Start(const Instruction *Start) {
148 if (Root.lookup(Start) == 0)
152 const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {
153 unsigned ComponentID = ValueToComponent.lookup(V);
155 assert(ComponentID > 0 &&
156 "Asking for a component for a value we never processed");
157 return Components[ComponentID];
161 void FindSCC(const Instruction *I) {
163 // Store the DFS Number we had before it possibly gets incremented.
164 unsigned int OurDFS = DFSNum;
165 for (auto &Op : I->operands()) {
166 if (auto *InstOp = dyn_cast<Instruction>(Op)) {
167 if (Root.lookup(Op) == 0)
169 if (!InComponent.count(Op))
170 Root[I] = std::min(Root.lookup(I), Root.lookup(Op));
173 // See if we really were the root of a component, by seeing if we still have our DFSNumber.
174 // If we do, we are the root of the component, and we have completed a component. If we do not,
175 // we are not the root of a component, and belong on the component stack.
176 if (Root.lookup(I) == OurDFS) {
177 unsigned ComponentID = Components.size();
178 Components.resize(Components.size() + 1);
179 auto &Component = Components.back();
181 DEBUG(dbgs() << "Component root is " << *I << "\n");
182 InComponent.insert(I);
183 ValueToComponent[I] = ComponentID;
184 // Pop a component off the stack and label it.
185 while (!Stack.empty() && Root.lookup(Stack.back()) >= OurDFS) {
186 auto *Member = Stack.back();
187 DEBUG(dbgs() << "Component member is " << *Member << "\n");
188 Component.insert(Member);
189 InComponent.insert(Member);
190 ValueToComponent[Member] = ComponentID;
194 // Part of a component, push to stack
198 unsigned int DFSNum = 1;
199 SmallPtrSet<const Value *, 8> InComponent;
200 DenseMap<const Value *, unsigned int> Root;
201 SmallVector<const Value *, 8> Stack;
202 // Store the components as vector of ptr sets, because we need the topo order
203 // of SCC's, but not individual member order
204 SmallVector<SmallPtrSet<const Value *, 8>, 8> Components;
205 DenseMap<const Value *, unsigned> ValueToComponent;
207 // Congruence classes represent the set of expressions/instructions
208 // that are all the same *during some scope in the function*.
209 // That is, because of the way we perform equality propagation, and
210 // because of memory value numbering, it is not correct to assume
211 // you can willy-nilly replace any member with any other at any
212 // point in the function.
214 // For any Value in the Member set, it is valid to replace any dominated member
217 // Every congruence class has a leader, and the leader is used to symbolize
218 // instructions in a canonical way (IE every operand of an instruction that is a
219 // member of the same congruence class will always be replaced with leader
220 // during symbolization). To simplify symbolization, we keep the leader as a
221 // constant if class can be proved to be a constant value. Otherwise, the
222 // leader is the member of the value set with the smallest DFS number. Each
223 // congruence class also has a defining expression, though the expression may be
224 // null. If it exists, it can be used for forward propagation and reassociation
227 // For memory, we also track a representative MemoryAccess, and a set of memory
228 // members for MemoryPhis (which have no real instructions). Note that for
229 // memory, it seems tempting to try to split the memory members into a
230 // MemoryCongruenceClass or something. Unfortunately, this does not work
231 // easily. The value numbering of a given memory expression depends on the
232 // leader of the memory congruence class, and the leader of memory congruence
233 // class depends on the value numbering of a given memory expression. This
234 // leads to wasted propagation, and in some cases, missed optimization. For
235 // example: If we had value numbered two stores together before, but now do not,
236 // we move them to a new value congruence class. This in turn will move at one
237 // of the memorydefs to a new memory congruence class. Which in turn, affects
238 // the value numbering of the stores we just value numbered (because the memory
239 // congruence class is part of the value number). So while theoretically
240 // possible to split them up, it turns out to be *incredibly* complicated to get
241 // it to work right, because of the interdependency. While structurally
242 // slightly messier, it is algorithmically much simpler and faster to do what we
243 // do here, and track them both at once in the same class.
244 // Note: The default iterators for this class iterate over values
245 class CongruenceClass {
247 using MemberType = Value;
248 using MemberSet = SmallPtrSet<MemberType *, 4>;
249 using MemoryMemberType = MemoryPhi;
250 using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;
252 explicit CongruenceClass(unsigned ID) : ID(ID) {}
253 CongruenceClass(unsigned ID, Value *Leader, const Expression *E)
254 : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
255 unsigned getID() const { return ID; }
256 // True if this class has no members left. This is mainly used for assertion
257 // purposes, and for skipping empty classes.
258 bool isDead() const {
259 // If it's both dead from a value perspective, and dead from a memory
260 // perspective, it's really dead.
261 return empty() && memory_empty();
264 Value *getLeader() const { return RepLeader; }
265 void setLeader(Value *Leader) { RepLeader = Leader; }
266 const std::pair<Value *, unsigned int> &getNextLeader() const {
269 void resetNextLeader() { NextLeader = {nullptr, ~0}; }
271 void addPossibleNextLeader(std::pair<Value *, unsigned int> LeaderPair) {
272 if (LeaderPair.second < NextLeader.second)
273 NextLeader = LeaderPair;
276 Value *getStoredValue() const { return RepStoredValue; }
277 void setStoredValue(Value *Leader) { RepStoredValue = Leader; }
278 const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }
279 void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }
281 // Forward propagation info
282 const Expression *getDefiningExpr() const { return DefiningExpr; }
283 void setDefiningExpr(const Expression *E) { DefiningExpr = E; }
286 bool empty() const { return Members.empty(); }
287 unsigned size() const { return Members.size(); }
288 MemberSet::const_iterator begin() const { return Members.begin(); }
289 MemberSet::const_iterator end() const { return Members.end(); }
290 void insert(MemberType *M) { Members.insert(M); }
291 void erase(MemberType *M) { Members.erase(M); }
292 void swap(MemberSet &Other) { Members.swap(Other); }
295 bool memory_empty() const { return MemoryMembers.empty(); }
296 unsigned memory_size() const { return MemoryMembers.size(); }
297 MemoryMemberSet::const_iterator memory_begin() const {
298 return MemoryMembers.begin();
300 MemoryMemberSet::const_iterator memory_end() const {
301 return MemoryMembers.end();
303 iterator_range<MemoryMemberSet::const_iterator> memory() const {
304 return make_range(memory_begin(), memory_end());
306 void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); }
307 void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); }
310 unsigned getStoreCount() const { return StoreCount; }
311 void incStoreCount() { ++StoreCount; }
312 void decStoreCount() {
313 assert(StoreCount != 0 && "Store count went negative");
317 // Return true if two congruence classes are equivalent to each other. This
319 // that every field but the ID number and the dead field are equivalent.
320 bool isEquivalentTo(const CongruenceClass *Other) const {
326 if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) !=
327 std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue,
328 Other->RepMemoryAccess))
330 if (DefiningExpr != Other->DefiningExpr)
331 if (!DefiningExpr || !Other->DefiningExpr ||
332 *DefiningExpr != *Other->DefiningExpr)
334 // We need some ordered set
335 std::set<Value *> AMembers(Members.begin(), Members.end());
336 std::set<Value *> BMembers(Members.begin(), Members.end());
337 return AMembers == BMembers;
342 // Representative leader.
343 Value *RepLeader = nullptr;
344 // The most dominating leader after our current leader, because the member set
345 // is not sorted and is expensive to keep sorted all the time.
346 std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
347 // If this is represented by a store, the value of the store.
348 Value *RepStoredValue = nullptr;
349 // If this class contains MemoryDefs or MemoryPhis, this is the leading memory
351 const MemoryAccess *RepMemoryAccess = nullptr;
352 // Defining Expression.
353 const Expression *DefiningExpr = nullptr;
354 // Actual members of this class.
356 // This is the set of MemoryPhis that exist in the class. MemoryDefs and
357 // MemoryUses have real instructions representing them, so we only need to
358 // track MemoryPhis here.
359 MemoryMemberSet MemoryMembers;
360 // Number of stores in this congruence class.
361 // This is used so we can detect store equivalence changes properly.
366 template <> struct DenseMapInfo<const Expression *> {
367 static const Expression *getEmptyKey() {
368 auto Val = static_cast<uintptr_t>(-1);
369 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
370 return reinterpret_cast<const Expression *>(Val);
372 static const Expression *getTombstoneKey() {
373 auto Val = static_cast<uintptr_t>(~1U);
374 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
375 return reinterpret_cast<const Expression *>(Val);
377 static unsigned getHashValue(const Expression *V) {
378 return static_cast<unsigned>(V->getHashValue());
380 static bool isEqual(const Expression *LHS, const Expression *RHS) {
383 if (LHS == getTombstoneKey() || RHS == getTombstoneKey() ||
384 LHS == getEmptyKey() || RHS == getEmptyKey())
389 } // end namespace llvm
396 const TargetLibraryInfo *TLI;
399 MemorySSAWalker *MSSAWalker;
400 const DataLayout &DL;
401 std::unique_ptr<PredicateInfo> PredInfo;
402 BumpPtrAllocator ExpressionAllocator;
403 ArrayRecycler<Value *> ArgRecycler;
406 // Number of function arguments, used by ranking
407 unsigned int NumFuncArgs;
409 // RPOOrdering of basic blocks
410 DenseMap<const DomTreeNode *, unsigned> RPOOrdering;
412 // Congruence class info.
414 // This class is called INITIAL in the paper. It is the class everything
415 // startsout in, and represents any value. Being an optimistic analysis,
416 // anything in the TOP class has the value TOP, which is indeterminate and
417 // equivalent to everything.
418 CongruenceClass *TOPClass;
419 std::vector<CongruenceClass *> CongruenceClasses;
420 unsigned NextCongruenceNum;
423 DenseMap<Value *, CongruenceClass *> ValueToClass;
424 DenseMap<Value *, const Expression *> ValueToExpression;
426 // Mapping from predicate info we used to the instructions we used it with.
427 // In order to correctly ensure propagation, we must keep track of what
428 // comparisons we used, so that when the values of the comparisons change, we
429 // propagate the information to the places we used the comparison.
430 DenseMap<const Value *, SmallPtrSet<Instruction *, 2>> PredicateToUsers;
431 // Mapping from MemoryAccess we used to the MemoryAccess we used it with. Has
432 // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for
433 // stores, we no longer can rely solely on the def-use chains of MemorySSA.
434 DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>> MemoryToUsers;
436 // A table storing which memorydefs/phis represent a memory state provably
437 // equivalent to another memory state.
438 // We could use the congruence class machinery, but the MemoryAccess's are
439 // abstract memory states, so they can only ever be equivalent to each other,
440 // and not to constants, etc.
441 DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass;
443 // We could, if we wanted, build MemoryPhiExpressions and
444 // MemoryVariableExpressions, etc, and value number them the same way we value
445 // number phi expressions. For the moment, this seems like overkill. They
446 // can only exist in one of three states: they can be TOP (equal to
447 // everything), Equivalent to something else, or unique. Because we do not
448 // create expressions for them, we need to simulate leader change not just
449 // when they change class, but when they change state. Note: We can do the
450 // same thing for phis, and avoid having phi expressions if we wanted, We
451 // should eventually unify in one direction or the other, so this is a little
452 // bit of an experiment in which turns out easier to maintain.
453 enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };
454 DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState;
456 enum PhiCycleState { PCS_Unknown, PCS_CycleFree, PCS_Cycle };
457 DenseMap<const PHINode *, PhiCycleState> PhiCycleState;
458 // Expression to class mapping.
459 using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
460 ExpressionClassMap ExpressionToClass;
462 // Which values have changed as a result of leader changes.
463 SmallPtrSet<Value *, 8> LeaderChanges;
465 // Reachability info.
466 using BlockEdge = BasicBlockEdge;
467 DenseSet<BlockEdge> ReachableEdges;
468 SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
470 // This is a bitvector because, on larger functions, we may have
471 // thousands of touched instructions at once (entire blocks,
472 // instructions with hundreds of uses, etc). Even with optimization
473 // for when we mark whole blocks as touched, when this was a
474 // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
475 // the time in GVN just managing this list. The bitvector, on the
476 // other hand, efficiently supports test/set/clear of both
477 // individual and ranges, as well as "find next element" This
478 // enables us to use it as a worklist with essentially 0 cost.
479 BitVector TouchedInstructions;
481 DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
484 // Debugging for how many times each block and instruction got processed.
485 DenseMap<const Value *, unsigned> ProcessedCount;
489 // This contains a mapping from Instructions to DFS numbers.
490 // The numbering starts at 1. An instruction with DFS number zero
491 // means that the instruction is dead.
492 DenseMap<const Value *, unsigned> InstrDFS;
494 // This contains the mapping DFS numbers to instructions.
495 SmallVector<Value *, 32> DFSToInstr;
498 SmallPtrSet<Instruction *, 8> InstructionsToErase;
501 NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
502 TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
503 const DataLayout &DL)
504 : F(F), DT(DT), AC(AC), TLI(TLI), AA(AA), MSSA(MSSA), DL(DL),
505 PredInfo(make_unique<PredicateInfo>(F, *DT, *AC)) {}
509 // Expression handling.
510 const Expression *createExpression(Instruction *);
511 const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *);
512 PHIExpression *createPHIExpression(Instruction *, bool &HasBackedge,
514 const VariableExpression *createVariableExpression(Value *);
515 const ConstantExpression *createConstantExpression(Constant *);
516 const Expression *createVariableOrConstant(Value *V);
517 const UnknownExpression *createUnknownExpression(Instruction *);
518 const StoreExpression *createStoreExpression(StoreInst *,
519 const MemoryAccess *);
520 LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
521 const MemoryAccess *);
522 const CallExpression *createCallExpression(CallInst *, const MemoryAccess *);
523 const AggregateValueExpression *createAggregateValueExpression(Instruction *);
524 bool setBasicExpressionInfo(Instruction *, BasicExpression *);
526 // Congruence class handling.
527 CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
528 auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E);
529 CongruenceClasses.emplace_back(result);
533 CongruenceClass *createMemoryClass(MemoryAccess *MA) {
534 auto *CC = createCongruenceClass(nullptr, nullptr);
535 CC->setMemoryLeader(MA);
538 CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {
539 auto *CC = getMemoryClass(MA);
540 if (CC->getMemoryLeader() != MA)
541 CC = createMemoryClass(MA);
545 CongruenceClass *createSingletonCongruenceClass(Value *Member) {
546 CongruenceClass *CClass = createCongruenceClass(Member, nullptr);
547 CClass->insert(Member);
548 ValueToClass[Member] = CClass;
551 void initializeCongruenceClasses(Function &F);
553 // Value number an Instruction or MemoryPhi.
554 void valueNumberMemoryPhi(MemoryPhi *);
555 void valueNumberInstruction(Instruction *);
557 // Symbolic evaluation.
558 const Expression *checkSimplificationResults(Expression *, Instruction *,
560 const Expression *performSymbolicEvaluation(Value *);
561 const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
562 Instruction *, MemoryAccess *);
563 const Expression *performSymbolicLoadEvaluation(Instruction *);
564 const Expression *performSymbolicStoreEvaluation(Instruction *);
565 const Expression *performSymbolicCallEvaluation(Instruction *);
566 const Expression *performSymbolicPHIEvaluation(Instruction *);
567 const Expression *performSymbolicAggrValueEvaluation(Instruction *);
568 const Expression *performSymbolicCmpEvaluation(Instruction *);
569 const Expression *performSymbolicPredicateInfoEvaluation(Instruction *);
571 // Congruence finding.
572 bool someEquivalentDominates(const Instruction *, const Instruction *) const;
573 Value *lookupOperandLeader(Value *) const;
574 void performCongruenceFinding(Instruction *, const Expression *);
575 void moveValueToNewCongruenceClass(Instruction *, const Expression *,
576 CongruenceClass *, CongruenceClass *);
577 void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,
578 CongruenceClass *, CongruenceClass *);
579 Value *getNextValueLeader(CongruenceClass *) const;
580 const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;
581 bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);
582 CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;
583 const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;
584 bool isMemoryAccessTop(const MemoryAccess *) const;
587 unsigned int getRank(const Value *) const;
588 bool shouldSwapOperands(const Value *, const Value *) const;
590 // Reachability handling.
591 void updateReachableEdge(BasicBlock *, BasicBlock *);
592 void processOutgoingEdges(TerminatorInst *, BasicBlock *);
593 Value *findConditionEquivalence(Value *) const;
597 void convertClassToDFSOrdered(const CongruenceClass &,
598 SmallVectorImpl<ValueDFS> &,
599 DenseMap<const Value *, unsigned int> &,
600 SmallPtrSetImpl<Instruction *> &) const;
601 void convertClassToLoadsAndStores(const CongruenceClass &,
602 SmallVectorImpl<ValueDFS> &) const;
604 bool eliminateInstructions(Function &);
605 void replaceInstruction(Instruction *, Value *);
606 void markInstructionForDeletion(Instruction *);
607 void deleteInstructionsInBlock(BasicBlock *);
609 // New instruction creation.
610 void handleNewInstruction(Instruction *){};
612 // Various instruction touch utilities
613 void markUsersTouched(Value *);
614 void markMemoryUsersTouched(const MemoryAccess *);
615 void markMemoryDefTouched(const MemoryAccess *);
616 void markPredicateUsersTouched(Instruction *);
617 void markValueLeaderChangeTouched(CongruenceClass *CC);
618 void markMemoryLeaderChangeTouched(CongruenceClass *CC);
619 void addPredicateUsers(const PredicateBase *, Instruction *);
620 void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U);
622 // Main loop of value numbering
623 void iterateTouchedInstructions();
626 void cleanupTables();
627 std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
628 void updateProcessedCount(Value *V);
629 void verifyMemoryCongruency() const;
630 void verifyIterationSettled(Function &F);
631 bool singleReachablePHIPath(const MemoryAccess *, const MemoryAccess *) const;
632 BasicBlock *getBlockForValue(Value *V) const;
633 void deleteExpression(const Expression *E);
634 unsigned InstrToDFSNum(const Value *V) const {
635 assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");
636 return InstrDFS.lookup(V);
639 unsigned InstrToDFSNum(const MemoryAccess *MA) const {
640 return MemoryToDFSNum(MA);
642 Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }
643 // Given a MemoryAccess, return the relevant instruction DFS number. Note:
644 // This deliberately takes a value so it can be used with Use's, which will
645 // auto-convert to Value's but not to MemoryAccess's.
646 unsigned MemoryToDFSNum(const Value *MA) const {
647 assert(isa<MemoryAccess>(MA) &&
648 "This should not be used with instructions");
649 return isa<MemoryUseOrDef>(MA)
650 ? InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst())
651 : InstrDFS.lookup(MA);
653 bool isCycleFree(const PHINode *PN);
654 template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
655 // Debug counter info. When verifying, we have to reset the value numbering
656 // debug counter to the same state it started in to get the same results.
657 std::pair<int, int> StartingVNCounter;
659 } // end anonymous namespace
661 template <typename T>
662 static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
663 if (!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS))
665 return LHS.MemoryExpression::equals(RHS);
668 bool LoadExpression::equals(const Expression &Other) const {
669 return equalsLoadStoreHelper(*this, Other);
672 bool StoreExpression::equals(const Expression &Other) const {
673 if (!equalsLoadStoreHelper(*this, Other))
675 // Make sure that store vs store includes the value operand.
676 if (const auto *S = dyn_cast<StoreExpression>(&Other))
677 if (getStoredValue() != S->getStoredValue())
683 static std::string getBlockName(const BasicBlock *B) {
684 return DOTGraphTraits<const Function *>::getSimpleNodeLabel(B, nullptr);
688 // Get the basic block from an instruction/memory value.
689 BasicBlock *NewGVN::getBlockForValue(Value *V) const {
690 if (auto *I = dyn_cast<Instruction>(V))
691 return I->getParent();
692 else if (auto *MP = dyn_cast<MemoryPhi>(V))
693 return MP->getBlock();
694 llvm_unreachable("Should have been able to figure out a block for our value");
698 // Delete a definitely dead expression, so it can be reused by the expression
699 // allocator. Some of these are not in creation functions, so we have to accept
701 void NewGVN::deleteExpression(const Expression *E) {
702 assert(isa<BasicExpression>(E));
703 auto *BE = cast<BasicExpression>(E);
704 const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler);
705 ExpressionAllocator.Deallocate(E);
708 PHIExpression *NewGVN::createPHIExpression(Instruction *I, bool &HasBackedge,
710 BasicBlock *PHIBlock = I->getParent();
711 auto *PN = cast<PHINode>(I);
713 new (ExpressionAllocator) PHIExpression(PN->getNumOperands(), PHIBlock);
715 E->allocateOperands(ArgRecycler, ExpressionAllocator);
716 E->setType(I->getType());
717 E->setOpcode(I->getOpcode());
719 unsigned PHIRPO = RPOOrdering.lookup(DT->getNode(PHIBlock));
721 // Filter out unreachable phi operands.
722 auto Filtered = make_filter_range(PN->operands(), [&](const Use &U) {
723 return ReachableEdges.count({PN->getIncomingBlock(U), PHIBlock});
726 std::transform(Filtered.begin(), Filtered.end(), op_inserter(E),
727 [&](const Use &U) -> Value * {
728 auto *BB = PN->getIncomingBlock(U);
729 auto *DTN = DT->getNode(BB);
730 if (RPOOrdering.lookup(DTN) >= PHIRPO)
732 AllConstant &= isa<UndefValue>(U) || isa<Constant>(U);
734 // Don't try to transform self-defined phis.
737 return lookupOperandLeader(U);
742 // Set basic expression info (Arguments, type, opcode) for Expression
743 // E from Instruction I in block B.
744 bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) {
745 bool AllConstant = true;
746 if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
747 E->setType(GEP->getSourceElementType());
749 E->setType(I->getType());
750 E->setOpcode(I->getOpcode());
751 E->allocateOperands(ArgRecycler, ExpressionAllocator);
753 // Transform the operand array into an operand leader array, and keep track of
754 // whether all members are constant.
755 std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {
756 auto Operand = lookupOperandLeader(O);
757 AllConstant &= isa<Constant>(Operand);
764 const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
765 Value *Arg1, Value *Arg2) {
766 auto *E = new (ExpressionAllocator) BasicExpression(2);
769 E->setOpcode(Opcode);
770 E->allocateOperands(ArgRecycler, ExpressionAllocator);
771 if (Instruction::isCommutative(Opcode)) {
772 // Ensure that commutative instructions that only differ by a permutation
773 // of their operands get the same value number by sorting the operand value
774 // numbers. Since all commutative instructions have two operands it is more
775 // efficient to sort by hand rather than using, say, std::sort.
776 if (shouldSwapOperands(Arg1, Arg2))
777 std::swap(Arg1, Arg2);
779 E->op_push_back(lookupOperandLeader(Arg1));
780 E->op_push_back(lookupOperandLeader(Arg2));
782 Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), DL, TLI,
784 if (const Expression *SimplifiedE = checkSimplificationResults(E, nullptr, V))
789 // Take a Value returned by simplification of Expression E/Instruction
790 // I, and see if it resulted in a simpler expression. If so, return
792 // TODO: Once finished, this should not take an Instruction, we only
793 // use it for printing.
794 const Expression *NewGVN::checkSimplificationResults(Expression *E,
795 Instruction *I, Value *V) {
798 if (auto *C = dyn_cast<Constant>(V)) {
800 DEBUG(dbgs() << "Simplified " << *I << " to "
801 << " constant " << *C << "\n");
802 NumGVNOpsSimplified++;
803 assert(isa<BasicExpression>(E) &&
804 "We should always have had a basic expression here");
806 return createConstantExpression(C);
807 } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
809 DEBUG(dbgs() << "Simplified " << *I << " to "
810 << " variable " << *V << "\n");
812 return createVariableExpression(V);
815 CongruenceClass *CC = ValueToClass.lookup(V);
816 if (CC && CC->getDefiningExpr()) {
818 DEBUG(dbgs() << "Simplified " << *I << " to "
819 << " expression " << *V << "\n");
820 NumGVNOpsSimplified++;
822 return CC->getDefiningExpr();
827 const Expression *NewGVN::createExpression(Instruction *I) {
828 auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
830 bool AllConstant = setBasicExpressionInfo(I, E);
832 if (I->isCommutative()) {
833 // Ensure that commutative instructions that only differ by a permutation
834 // of their operands get the same value number by sorting the operand value
835 // numbers. Since all commutative instructions have two operands it is more
836 // efficient to sort by hand rather than using, say, std::sort.
837 assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
838 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))
839 E->swapOperands(0, 1);
842 // Perform simplificaiton
843 // TODO: Right now we only check to see if we get a constant result.
844 // We may get a less than constant, but still better, result for
849 // We should handle this by simply rewriting the expression.
850 if (auto *CI = dyn_cast<CmpInst>(I)) {
851 // Sort the operand value numbers so x<y and y>x get the same value
853 CmpInst::Predicate Predicate = CI->getPredicate();
854 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) {
855 E->swapOperands(0, 1);
856 Predicate = CmpInst::getSwappedPredicate(Predicate);
858 E->setOpcode((CI->getOpcode() << 8) | Predicate);
859 // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands
860 assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
861 "Wrong types on cmp instruction");
862 assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
863 E->getOperand(1)->getType() == I->getOperand(1)->getType()));
864 Value *V = SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1),
866 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
868 } else if (isa<SelectInst>(I)) {
869 if (isa<Constant>(E->getOperand(0)) ||
870 E->getOperand(0) == E->getOperand(1)) {
871 assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
872 E->getOperand(2)->getType() == I->getOperand(2)->getType());
873 Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
874 E->getOperand(2), DL, TLI, DT, AC);
875 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
878 } else if (I->isBinaryOp()) {
879 Value *V = SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1),
881 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
883 } else if (auto *BI = dyn_cast<BitCastInst>(I)) {
884 Value *V = SimplifyInstruction(BI, DL, TLI, DT, AC);
885 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
887 } else if (isa<GetElementPtrInst>(I)) {
888 Value *V = SimplifyGEPInst(E->getType(),
889 ArrayRef<Value *>(E->op_begin(), E->op_end()),
891 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
893 } else if (AllConstant) {
894 // We don't bother trying to simplify unless all of the operands
896 // TODO: There are a lot of Simplify*'s we could call here, if we
897 // wanted to. The original motivating case for this code was a
898 // zext i1 false to i8, which we don't have an interface to
899 // simplify (IE there is no SimplifyZExt).
901 SmallVector<Constant *, 8> C;
902 for (Value *Arg : E->operands())
903 C.emplace_back(cast<Constant>(Arg));
905 if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))
906 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
912 const AggregateValueExpression *
913 NewGVN::createAggregateValueExpression(Instruction *I) {
914 if (auto *II = dyn_cast<InsertValueInst>(I)) {
915 auto *E = new (ExpressionAllocator)
916 AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
917 setBasicExpressionInfo(I, E);
918 E->allocateIntOperands(ExpressionAllocator);
919 std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E));
921 } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
922 auto *E = new (ExpressionAllocator)
923 AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
924 setBasicExpressionInfo(EI, E);
925 E->allocateIntOperands(ExpressionAllocator);
926 std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E));
929 llvm_unreachable("Unhandled type of aggregate value operation");
932 const VariableExpression *NewGVN::createVariableExpression(Value *V) {
933 auto *E = new (ExpressionAllocator) VariableExpression(V);
934 E->setOpcode(V->getValueID());
938 const Expression *NewGVN::createVariableOrConstant(Value *V) {
939 if (auto *C = dyn_cast<Constant>(V))
940 return createConstantExpression(C);
941 return createVariableExpression(V);
944 const ConstantExpression *NewGVN::createConstantExpression(Constant *C) {
945 auto *E = new (ExpressionAllocator) ConstantExpression(C);
946 E->setOpcode(C->getValueID());
950 const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) {
951 auto *E = new (ExpressionAllocator) UnknownExpression(I);
952 E->setOpcode(I->getOpcode());
956 const CallExpression *NewGVN::createCallExpression(CallInst *CI,
957 const MemoryAccess *MA) {
958 // FIXME: Add operand bundles for calls.
960 new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);
961 setBasicExpressionInfo(CI, E);
965 // Return true if some equivalent of instruction Inst dominates instruction U.
966 bool NewGVN::someEquivalentDominates(const Instruction *Inst,
967 const Instruction *U) const {
968 auto *CC = ValueToClass.lookup(Inst);
969 // This must be an instruction because we are only called from phi nodes
970 // in the case that the value it needs to check against is an instruction.
972 // The most likely candiates for dominance are the leader and the next leader.
973 // The leader or nextleader will dominate in all cases where there is an
974 // equivalent that is higher up in the dom tree.
975 // We can't *only* check them, however, because the
976 // dominator tree could have an infinite number of non-dominating siblings
977 // with instructions that are in the right congruence class.
982 // Instruction U could be in H, with equivalents in every other sibling.
983 // Depending on the rpo order picked, the leader could be the equivalent in
984 // any of these siblings.
987 if (DT->dominates(cast<Instruction>(CC->getLeader()), U))
989 if (CC->getNextLeader().first &&
990 DT->dominates(cast<Instruction>(CC->getNextLeader().first), U))
992 return llvm::any_of(*CC, [&](const Value *Member) {
993 return Member != CC->getLeader() &&
994 DT->dominates(cast<Instruction>(Member), U);
998 // See if we have a congruence class and leader for this operand, and if so,
999 // return it. Otherwise, return the operand itself.
1000 Value *NewGVN::lookupOperandLeader(Value *V) const {
1001 CongruenceClass *CC = ValueToClass.lookup(V);
1003 // Everything in TOP is represneted by undef, as it can be any value.
1004 // We do have to make sure we get the type right though, so we can't set the
1005 // RepLeader to undef.
1007 return UndefValue::get(V->getType());
1008 return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
1014 const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {
1015 auto *CC = getMemoryClass(MA);
1016 assert(CC->getMemoryLeader() &&
1017 "Every MemoryAccess should be mapped to a "
1018 "congruence class with a represenative memory "
1020 return CC->getMemoryLeader();
1023 // Return true if the MemoryAccess is really equivalent to everything. This is
1024 // equivalent to the lattice value "TOP" in most lattices. This is the initial
1025 // state of all MemoryAccesses.
1026 bool NewGVN::isMemoryAccessTop(const MemoryAccess *MA) const {
1027 return getMemoryClass(MA) == TOPClass;
1030 LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
1032 const MemoryAccess *MA) {
1034 new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));
1035 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1036 E->setType(LoadType);
1038 // Give store and loads same opcode so they value number together.
1040 E->op_push_back(PointerOp);
1042 E->setAlignment(LI->getAlignment());
1044 // TODO: Value number heap versions. We may be able to discover
1045 // things alias analysis can't on it's own (IE that a store and a
1046 // load have the same value, and thus, it isn't clobbering the load).
1050 const StoreExpression *NewGVN::createStoreExpression(StoreInst *SI,
1051 const MemoryAccess *MA) {
1052 auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand());
1053 auto *E = new (ExpressionAllocator)
1054 StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);
1055 E->allocateOperands(ArgRecycler, ExpressionAllocator);
1056 E->setType(SI->getValueOperand()->getType());
1058 // Give store and loads same opcode so they value number together.
1060 E->op_push_back(lookupOperandLeader(SI->getPointerOperand()));
1062 // TODO: Value number heap versions. We may be able to discover
1063 // things alias analysis can't on it's own (IE that a store and a
1064 // load have the same value, and thus, it isn't clobbering the load).
1068 const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) {
1069 // Unlike loads, we never try to eliminate stores, so we do not check if they
1070 // are simple and avoid value numbering them.
1071 auto *SI = cast<StoreInst>(I);
1072 auto *StoreAccess = MSSA->getMemoryAccess(SI);
1073 // Get the expression, if any, for the RHS of the MemoryDef.
1074 const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();
1075 if (EnableStoreRefinement)
1076 StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess);
1077 // If we bypassed the use-def chains, make sure we add a use.
1078 if (StoreRHS != StoreAccess->getDefiningAccess())
1079 addMemoryUsers(StoreRHS, StoreAccess);
1081 StoreRHS = lookupMemoryLeader(StoreRHS);
1082 // If we are defined by ourselves, use the live on entry def.
1083 if (StoreRHS == StoreAccess)
1084 StoreRHS = MSSA->getLiveOnEntryDef();
1086 if (SI->isSimple()) {
1087 // See if we are defined by a previous store expression, it already has a
1088 // value, and it's the same value as our current store. FIXME: Right now, we
1089 // only do this for simple stores, we should expand to cover memcpys, etc.
1090 const auto *LastStore = createStoreExpression(SI, StoreRHS);
1091 const auto *LastCC = ExpressionToClass.lookup(LastStore);
1092 // Basically, check if the congruence class the store is in is defined by a
1093 // store that isn't us, and has the same value. MemorySSA takes care of
1094 // ensuring the store has the same memory state as us already.
1095 // The RepStoredValue gets nulled if all the stores disappear in a class, so
1096 // we don't need to check if the class contains a store besides us.
1098 LastCC->getStoredValue() == lookupOperandLeader(SI->getValueOperand()))
1100 deleteExpression(LastStore);
1101 // Also check if our value operand is defined by a load of the same memory
1102 // location, and the memory state is the same as it was then (otherwise, it
1103 // could have been overwritten later. See test32 in
1104 // transforms/DeadStoreElimination/simple.ll).
1106 dyn_cast<LoadInst>(lookupOperandLeader(SI->getValueOperand()))) {
1107 if ((lookupOperandLeader(LI->getPointerOperand()) ==
1108 lookupOperandLeader(SI->getPointerOperand())) &&
1109 (lookupMemoryLeader(MSSA->getMemoryAccess(LI)->getDefiningAccess()) ==
1111 return createVariableExpression(LI);
1115 // If the store is not equivalent to anything, value number it as a store that
1116 // produces a unique memory state (instead of using it's MemoryUse, we use
1118 return createStoreExpression(SI, StoreAccess);
1121 // See if we can extract the value of a loaded pointer from a load, a store, or
1122 // a memory instruction.
1124 NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
1125 LoadInst *LI, Instruction *DepInst,
1126 MemoryAccess *DefiningAccess) {
1127 assert((!LI || LI->isSimple()) && "Not a simple load");
1128 if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {
1129 // Can't forward from non-atomic to atomic without violating memory model.
1130 // Also don't need to coerce if they are the same type, we will just
1132 if (LI->isAtomic() > DepSI->isAtomic() ||
1133 LoadType == DepSI->getValueOperand()->getType())
1135 int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL);
1137 if (auto *C = dyn_cast<Constant>(
1138 lookupOperandLeader(DepSI->getValueOperand()))) {
1139 DEBUG(dbgs() << "Coercing load from store " << *DepSI << " to constant "
1141 return createConstantExpression(
1142 getConstantStoreValueForLoad(C, Offset, LoadType, DL));
1146 } else if (LoadInst *DepLI = dyn_cast<LoadInst>(DepInst)) {
1147 // Can't forward from non-atomic to atomic without violating memory model.
1148 if (LI->isAtomic() > DepLI->isAtomic())
1150 int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);
1152 // We can coerce a constant load into a load
1153 if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))
1154 if (auto *PossibleConstant =
1155 getConstantLoadValueForLoad(C, Offset, LoadType, DL)) {
1156 DEBUG(dbgs() << "Coercing load from load " << *LI << " to constant "
1157 << *PossibleConstant << "\n");
1158 return createConstantExpression(PossibleConstant);
1162 } else if (MemIntrinsic *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
1163 int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);
1165 if (auto *PossibleConstant =
1166 getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) {
1167 DEBUG(dbgs() << "Coercing load from meminst " << *DepMI
1168 << " to constant " << *PossibleConstant << "\n");
1169 return createConstantExpression(PossibleConstant);
1174 // All of the below are only true if the loaded pointer is produced
1175 // by the dependent instruction.
1176 if (LoadPtr != lookupOperandLeader(DepInst) &&
1177 !AA->isMustAlias(LoadPtr, DepInst))
1179 // If this load really doesn't depend on anything, then we must be loading an
1180 // undef value. This can happen when loading for a fresh allocation with no
1181 // intervening stores, for example. Note that this is only true in the case
1182 // that the result of the allocation is pointer equal to the load ptr.
1183 if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
1184 return createConstantExpression(UndefValue::get(LoadType));
1186 // If this load occurs either right after a lifetime begin,
1187 // then the loaded value is undefined.
1188 else if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {
1189 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1190 return createConstantExpression(UndefValue::get(LoadType));
1192 // If this load follows a calloc (which zero initializes memory),
1193 // then the loaded value is zero
1194 else if (isCallocLikeFn(DepInst, TLI)) {
1195 return createConstantExpression(Constant::getNullValue(LoadType));
1201 const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) {
1202 auto *LI = cast<LoadInst>(I);
1204 // We can eliminate in favor of non-simple loads, but we won't be able to
1205 // eliminate the loads themselves.
1206 if (!LI->isSimple())
1209 Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());
1210 // Load of undef is undef.
1211 if (isa<UndefValue>(LoadAddressLeader))
1212 return createConstantExpression(UndefValue::get(LI->getType()));
1214 MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(I);
1216 if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
1217 if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
1218 Instruction *DefiningInst = MD->getMemoryInst();
1219 // If the defining instruction is not reachable, replace with undef.
1220 if (!ReachableBlocks.count(DefiningInst->getParent()))
1221 return createConstantExpression(UndefValue::get(LI->getType()));
1222 // This will handle stores and memory insts. We only do if it the
1223 // defining access has a different type, or it is a pointer produced by
1224 // certain memory operations that cause the memory to have a fixed value
1225 // (IE things like calloc).
1226 if (const auto *CoercionResult =
1227 performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,
1228 DefiningInst, DefiningAccess))
1229 return CoercionResult;
1233 const Expression *E = createLoadExpression(LI->getType(), LoadAddressLeader,
1234 LI, DefiningAccess);
1239 NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) {
1240 auto *PI = PredInfo->getPredicateInfoFor(I);
1244 DEBUG(dbgs() << "Found predicate info from instruction !\n");
1246 auto *PWC = dyn_cast<PredicateWithCondition>(PI);
1250 auto *CopyOf = I->getOperand(0);
1251 auto *Cond = PWC->Condition;
1253 // If this a copy of the condition, it must be either true or false depending
1254 // on the predicate info type and edge
1255 if (CopyOf == Cond) {
1256 // We should not need to add predicate users because the predicate info is
1257 // already a use of this operand.
1258 if (isa<PredicateAssume>(PI))
1259 return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1260 if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1261 if (PBranch->TrueEdge)
1262 return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1263 return createConstantExpression(ConstantInt::getFalse(Cond->getType()));
1265 if (auto *PSwitch = dyn_cast<PredicateSwitch>(PI))
1266 return createConstantExpression(cast<Constant>(PSwitch->CaseValue));
1269 // Not a copy of the condition, so see what the predicates tell us about this
1270 // value. First, though, we check to make sure the value is actually a copy
1271 // of one of the condition operands. It's possible, in certain cases, for it
1272 // to be a copy of a predicateinfo copy. In particular, if two branch
1273 // operations use the same condition, and one branch dominates the other, we
1274 // will end up with a copy of a copy. This is currently a small deficiency in
1275 // predicateinfo. What will end up happening here is that we will value
1276 // number both copies the same anyway.
1278 // Everything below relies on the condition being a comparison.
1279 auto *Cmp = dyn_cast<CmpInst>(Cond);
1283 if (CopyOf != Cmp->getOperand(0) && CopyOf != Cmp->getOperand(1)) {
1284 DEBUG(dbgs() << "Copy is not of any condition operands!");
1287 Value *FirstOp = lookupOperandLeader(Cmp->getOperand(0));
1288 Value *SecondOp = lookupOperandLeader(Cmp->getOperand(1));
1289 bool SwappedOps = false;
1291 if (shouldSwapOperands(FirstOp, SecondOp)) {
1292 std::swap(FirstOp, SecondOp);
1295 CmpInst::Predicate Predicate =
1296 SwappedOps ? Cmp->getSwappedPredicate() : Cmp->getPredicate();
1298 if (isa<PredicateAssume>(PI)) {
1299 // If the comparison is true when the operands are equal, then we know the
1300 // operands are equal, because assumes must always be true.
1301 if (CmpInst::isTrueWhenEqual(Predicate)) {
1302 addPredicateUsers(PI, I);
1303 return createVariableOrConstant(FirstOp);
1306 if (const auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1307 // If we are *not* a copy of the comparison, we may equal to the other
1308 // operand when the predicate implies something about equality of
1309 // operations. In particular, if the comparison is true/false when the
1310 // operands are equal, and we are on the right edge, we know this operation
1311 // is equal to something.
1312 if ((PBranch->TrueEdge && Predicate == CmpInst::ICMP_EQ) ||
1313 (!PBranch->TrueEdge && Predicate == CmpInst::ICMP_NE)) {
1314 addPredicateUsers(PI, I);
1315 return createVariableOrConstant(FirstOp);
1317 // Handle the special case of floating point.
1318 if (((PBranch->TrueEdge && Predicate == CmpInst::FCMP_OEQ) ||
1319 (!PBranch->TrueEdge && Predicate == CmpInst::FCMP_UNE)) &&
1320 isa<ConstantFP>(FirstOp) && !cast<ConstantFP>(FirstOp)->isZero()) {
1321 addPredicateUsers(PI, I);
1322 return createConstantExpression(cast<Constant>(FirstOp));
1328 // Evaluate read only and pure calls, and create an expression result.
1329 const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) {
1330 auto *CI = cast<CallInst>(I);
1331 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1332 // Instrinsics with the returned attribute are copies of arguments.
1333 if (auto *ReturnedValue = II->getReturnedArgOperand()) {
1334 if (II->getIntrinsicID() == Intrinsic::ssa_copy)
1335 if (const auto *Result = performSymbolicPredicateInfoEvaluation(I))
1337 return createVariableOrConstant(ReturnedValue);
1340 if (AA->doesNotAccessMemory(CI)) {
1341 return createCallExpression(CI, TOPClass->getMemoryLeader());
1342 } else if (AA->onlyReadsMemory(CI)) {
1343 MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI);
1344 return createCallExpression(CI, DefiningAccess);
1349 // Retrieve the memory class for a given MemoryAccess.
1350 CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {
1352 auto *Result = MemoryAccessToClass.lookup(MA);
1353 assert(Result && "Should have found memory class");
1357 // Update the MemoryAccess equivalence table to say that From is equal to To,
1358 // and return true if this is different from what already existed in the table.
1359 bool NewGVN::setMemoryClass(const MemoryAccess *From,
1360 CongruenceClass *NewClass) {
1362 "Every MemoryAccess should be getting mapped to a non-null class");
1363 DEBUG(dbgs() << "Setting " << *From);
1364 DEBUG(dbgs() << " equivalent to congruence class ");
1365 DEBUG(dbgs() << NewClass->getID() << " with current MemoryAccess leader ");
1366 DEBUG(dbgs() << *NewClass->getMemoryLeader());
1367 DEBUG(dbgs() << "\n");
1369 auto LookupResult = MemoryAccessToClass.find(From);
1370 bool Changed = false;
1371 // If it's already in the table, see if the value changed.
1372 if (LookupResult != MemoryAccessToClass.end()) {
1373 auto *OldClass = LookupResult->second;
1374 if (OldClass != NewClass) {
1375 // If this is a phi, we have to handle memory member updates.
1376 if (auto *MP = dyn_cast<MemoryPhi>(From)) {
1377 OldClass->memory_erase(MP);
1378 NewClass->memory_insert(MP);
1379 // This may have killed the class if it had no non-memory members
1380 if (OldClass->getMemoryLeader() == From) {
1381 if (OldClass->memory_empty()) {
1382 OldClass->setMemoryLeader(nullptr);
1384 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1385 DEBUG(dbgs() << "Memory class leader change for class "
1386 << OldClass->getID() << " to "
1387 << *OldClass->getMemoryLeader()
1388 << " due to removal of a memory member " << *From
1390 markMemoryLeaderChangeTouched(OldClass);
1394 // It wasn't equivalent before, and now it is.
1395 LookupResult->second = NewClass;
1403 // Determine if a phi is cycle-free. That means the values in the phi don't
1404 // depend on any expressions that can change value as a result of the phi.
1405 // For example, a non-cycle free phi would be v = phi(0, v+1).
1406 bool NewGVN::isCycleFree(const PHINode *PN) {
1407 // In order to compute cycle-freeness, we do SCC finding on the phi, and see
1408 // what kind of SCC it ends up in. If it is a singleton, it is cycle-free.
1409 // If it is not in a singleton, it is only cycle free if the other members are
1410 // all phi nodes (as they do not compute anything, they are copies). TODO:
1411 // There are likely a few other intrinsics or expressions that could be
1412 // included here, but this happens so infrequently already that it is not
1413 // likely to be worth it.
1414 auto PCS = PhiCycleState.lookup(PN);
1415 if (PCS == PCS_Unknown) {
1416 SCCFinder.Start(PN);
1417 auto &SCC = SCCFinder.getComponentFor(PN);
1418 // It's cycle free if it's size 1 or or the SCC is *only* phi nodes.
1419 if (SCC.size() == 1)
1420 PhiCycleState.insert({PN, PCS_CycleFree});
1423 llvm::all_of(SCC, [](const Value *V) { return isa<PHINode>(V); });
1424 PCS = AllPhis ? PCS_CycleFree : PCS_Cycle;
1425 for (auto *Member : SCC)
1426 if (auto *MemberPhi = dyn_cast<PHINode>(Member))
1427 PhiCycleState.insert({MemberPhi, PCS});
1430 if (PCS == PCS_Cycle)
1435 // Evaluate PHI nodes symbolically, and create an expression result.
1436 const Expression *NewGVN::performSymbolicPHIEvaluation(Instruction *I) {
1437 // True if one of the incoming phi edges is a backedge.
1438 bool HasBackedge = false;
1439 // All constant tracks the state of whether all the *original* phi operands
1441 // This is really shorthand for "this phi cannot cycle due to forward
1442 // propagation", as any
1443 // change in value of the phi is guaranteed not to later change the value of
1445 // IE it can't be v = phi(undef, v+1)
1446 bool AllConstant = true;
1448 cast<PHIExpression>(createPHIExpression(I, HasBackedge, AllConstant));
1449 // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
1451 // See if all arguaments are the same.
1452 // We track if any were undef because they need special handling.
1453 bool HasUndef = false;
1454 auto Filtered = make_filter_range(E->operands(), [&](const Value *Arg) {
1457 if (isa<UndefValue>(Arg)) {
1463 // If we are left with no operands, it's undef
1464 if (Filtered.begin() == Filtered.end()) {
1465 DEBUG(dbgs() << "Simplified PHI node " << *I << " to undef"
1467 deleteExpression(E);
1468 return createConstantExpression(UndefValue::get(I->getType()));
1470 unsigned NumOps = 0;
1471 Value *AllSameValue = *(Filtered.begin());
1473 // Can't use std::equal here, sadly, because filter.begin moves.
1474 if (llvm::all_of(Filtered, [AllSameValue, &NumOps](const Value *V) {
1476 return V == AllSameValue;
1478 // In LLVM's non-standard representation of phi nodes, it's possible to have
1479 // phi nodes with cycles (IE dependent on other phis that are .... dependent
1480 // on the original phi node), especially in weird CFG's where some arguments
1481 // are unreachable, or uninitialized along certain paths. This can cause
1482 // infinite loops during evaluation. We work around this by not trying to
1483 // really evaluate them independently, but instead using a variable
1484 // expression to say if one is equivalent to the other.
1485 // We also special case undef, so that if we have an undef, we can't use the
1486 // common value unless it dominates the phi block.
1488 // If we have undef and at least one other value, this is really a
1489 // multivalued phi, and we need to know if it's cycle free in order to
1490 // evaluate whether we can ignore the undef. The other parts of this are
1491 // just shortcuts. If there is no backedge, or all operands are
1492 // constants, or all operands are ignored but the undef, it also must be
1494 if (!AllConstant && HasBackedge && NumOps > 0 &&
1495 !isa<UndefValue>(AllSameValue) && !isCycleFree(cast<PHINode>(I)))
1498 // Only have to check for instructions
1499 if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))
1500 if (!someEquivalentDominates(AllSameInst, I))
1504 NumGVNPhisAllSame++;
1505 DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
1507 deleteExpression(E);
1508 return createVariableOrConstant(AllSameValue);
1513 const Expression *NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) {
1514 if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1515 auto *II = dyn_cast<IntrinsicInst>(EI->getAggregateOperand());
1516 if (II && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) {
1517 unsigned Opcode = 0;
1518 // EI might be an extract from one of our recognised intrinsics. If it
1519 // is we'll synthesize a semantically equivalent expression instead on
1520 // an extract value expression.
1521 switch (II->getIntrinsicID()) {
1522 case Intrinsic::sadd_with_overflow:
1523 case Intrinsic::uadd_with_overflow:
1524 Opcode = Instruction::Add;
1526 case Intrinsic::ssub_with_overflow:
1527 case Intrinsic::usub_with_overflow:
1528 Opcode = Instruction::Sub;
1530 case Intrinsic::smul_with_overflow:
1531 case Intrinsic::umul_with_overflow:
1532 Opcode = Instruction::Mul;
1539 // Intrinsic recognized. Grab its args to finish building the
1541 assert(II->getNumArgOperands() == 2 &&
1542 "Expect two args for recognised intrinsics.");
1543 return createBinaryExpression(
1544 Opcode, EI->getType(), II->getArgOperand(0), II->getArgOperand(1));
1549 return createAggregateValueExpression(I);
1551 const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) {
1552 auto *CI = dyn_cast<CmpInst>(I);
1553 // See if our operands are equal to those of a previous predicate, and if so,
1554 // if it implies true or false.
1555 auto Op0 = lookupOperandLeader(CI->getOperand(0));
1556 auto Op1 = lookupOperandLeader(CI->getOperand(1));
1557 auto OurPredicate = CI->getPredicate();
1558 if (shouldSwapOperands(Op0, Op1)) {
1559 std::swap(Op0, Op1);
1560 OurPredicate = CI->getSwappedPredicate();
1563 // Avoid processing the same info twice
1564 const PredicateBase *LastPredInfo = nullptr;
1565 // See if we know something about the comparison itself, like it is the target
1567 auto *CmpPI = PredInfo->getPredicateInfoFor(I);
1568 if (dyn_cast_or_null<PredicateAssume>(CmpPI))
1569 return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1572 // This condition does not depend on predicates, no need to add users
1573 if (CI->isTrueWhenEqual())
1574 return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1575 else if (CI->isFalseWhenEqual())
1576 return createConstantExpression(ConstantInt::getFalse(CI->getType()));
1579 // NOTE: Because we are comparing both operands here and below, and using
1580 // previous comparisons, we rely on fact that predicateinfo knows to mark
1581 // comparisons that use renamed operands as users of the earlier comparisons.
1582 // It is *not* enough to just mark predicateinfo renamed operands as users of
1583 // the earlier comparisons, because the *other* operand may have changed in a
1584 // previous iteration.
1587 // %b.0 = ssa.copy(%b)
1589 // icmp slt %c, %b.0
1591 // %c and %a may start out equal, and thus, the code below will say the second
1592 // %icmp is false. c may become equal to something else, and in that case the
1593 // %second icmp *must* be reexamined, but would not if only the renamed
1594 // %operands are considered users of the icmp.
1596 // *Currently* we only check one level of comparisons back, and only mark one
1597 // level back as touched when changes appen . If you modify this code to look
1598 // back farther through comparisons, you *must* mark the appropriate
1599 // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if
1600 // we know something just from the operands themselves
1602 // See if our operands have predicate info, so that we may be able to derive
1603 // something from a previous comparison.
1604 for (const auto &Op : CI->operands()) {
1605 auto *PI = PredInfo->getPredicateInfoFor(Op);
1606 if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) {
1607 if (PI == LastPredInfo)
1611 // TODO: Along the false edge, we may know more things too, like icmp of
1612 // same operands is false.
1613 // TODO: We only handle actual comparison conditions below, not and/or.
1614 auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition);
1617 auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0));
1618 auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1));
1619 auto BranchPredicate = BranchCond->getPredicate();
1620 if (shouldSwapOperands(BranchOp0, BranchOp1)) {
1621 std::swap(BranchOp0, BranchOp1);
1622 BranchPredicate = BranchCond->getSwappedPredicate();
1624 if (BranchOp0 == Op0 && BranchOp1 == Op1) {
1625 if (PBranch->TrueEdge) {
1626 // If we know the previous predicate is true and we are in the true
1627 // edge then we may be implied true or false.
1628 if (CmpInst::isImpliedTrueByMatchingCmp(OurPredicate,
1630 addPredicateUsers(PI, I);
1631 return createConstantExpression(
1632 ConstantInt::getTrue(CI->getType()));
1635 if (CmpInst::isImpliedFalseByMatchingCmp(OurPredicate,
1637 addPredicateUsers(PI, I);
1638 return createConstantExpression(
1639 ConstantInt::getFalse(CI->getType()));
1643 // Just handle the ne and eq cases, where if we have the same
1644 // operands, we may know something.
1645 if (BranchPredicate == OurPredicate) {
1646 addPredicateUsers(PI, I);
1647 // Same predicate, same ops,we know it was false, so this is false.
1648 return createConstantExpression(
1649 ConstantInt::getFalse(CI->getType()));
1650 } else if (BranchPredicate ==
1651 CmpInst::getInversePredicate(OurPredicate)) {
1652 addPredicateUsers(PI, I);
1653 // Inverse predicate, we know the other was false, so this is true.
1654 return createConstantExpression(
1655 ConstantInt::getTrue(CI->getType()));
1661 // Create expression will take care of simplifyCmpInst
1662 return createExpression(I);
1665 // Substitute and symbolize the value before value numbering.
1666 const Expression *NewGVN::performSymbolicEvaluation(Value *V) {
1667 const Expression *E = nullptr;
1668 if (auto *C = dyn_cast<Constant>(V))
1669 E = createConstantExpression(C);
1670 else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
1671 E = createVariableExpression(V);
1673 // TODO: memory intrinsics.
1674 // TODO: Some day, we should do the forward propagation and reassociation
1675 // parts of the algorithm.
1676 auto *I = cast<Instruction>(V);
1677 switch (I->getOpcode()) {
1678 case Instruction::ExtractValue:
1679 case Instruction::InsertValue:
1680 E = performSymbolicAggrValueEvaluation(I);
1682 case Instruction::PHI:
1683 E = performSymbolicPHIEvaluation(I);
1685 case Instruction::Call:
1686 E = performSymbolicCallEvaluation(I);
1688 case Instruction::Store:
1689 E = performSymbolicStoreEvaluation(I);
1691 case Instruction::Load:
1692 E = performSymbolicLoadEvaluation(I);
1694 case Instruction::BitCast: {
1695 E = createExpression(I);
1697 case Instruction::ICmp:
1698 case Instruction::FCmp: {
1699 E = performSymbolicCmpEvaluation(I);
1701 case Instruction::Add:
1702 case Instruction::FAdd:
1703 case Instruction::Sub:
1704 case Instruction::FSub:
1705 case Instruction::Mul:
1706 case Instruction::FMul:
1707 case Instruction::UDiv:
1708 case Instruction::SDiv:
1709 case Instruction::FDiv:
1710 case Instruction::URem:
1711 case Instruction::SRem:
1712 case Instruction::FRem:
1713 case Instruction::Shl:
1714 case Instruction::LShr:
1715 case Instruction::AShr:
1716 case Instruction::And:
1717 case Instruction::Or:
1718 case Instruction::Xor:
1719 case Instruction::Trunc:
1720 case Instruction::ZExt:
1721 case Instruction::SExt:
1722 case Instruction::FPToUI:
1723 case Instruction::FPToSI:
1724 case Instruction::UIToFP:
1725 case Instruction::SIToFP:
1726 case Instruction::FPTrunc:
1727 case Instruction::FPExt:
1728 case Instruction::PtrToInt:
1729 case Instruction::IntToPtr:
1730 case Instruction::Select:
1731 case Instruction::ExtractElement:
1732 case Instruction::InsertElement:
1733 case Instruction::ShuffleVector:
1734 case Instruction::GetElementPtr:
1735 E = createExpression(I);
1744 void NewGVN::markUsersTouched(Value *V) {
1745 // Now mark the users as touched.
1746 for (auto *User : V->users()) {
1747 assert(isa<Instruction>(User) && "Use of value not within an instruction?");
1748 TouchedInstructions.set(InstrToDFSNum(User));
1752 void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) {
1753 DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");
1754 MemoryToUsers[To].insert(U);
1757 void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {
1758 TouchedInstructions.set(MemoryToDFSNum(MA));
1761 void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
1762 if (isa<MemoryUse>(MA))
1764 for (auto U : MA->users())
1765 TouchedInstructions.set(MemoryToDFSNum(U));
1766 const auto Result = MemoryToUsers.find(MA);
1767 if (Result != MemoryToUsers.end()) {
1768 for (auto *User : Result->second)
1769 TouchedInstructions.set(MemoryToDFSNum(User));
1770 MemoryToUsers.erase(Result);
1774 // Add I to the set of users of a given predicate.
1775 void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) {
1776 if (auto *PBranch = dyn_cast<PredicateBranch>(PB))
1777 PredicateToUsers[PBranch->Condition].insert(I);
1778 else if (auto *PAssume = dyn_cast<PredicateBranch>(PB))
1779 PredicateToUsers[PAssume->Condition].insert(I);
1782 // Touch all the predicates that depend on this instruction.
1783 void NewGVN::markPredicateUsersTouched(Instruction *I) {
1784 const auto Result = PredicateToUsers.find(I);
1785 if (Result != PredicateToUsers.end()) {
1786 for (auto *User : Result->second)
1787 TouchedInstructions.set(InstrToDFSNum(User));
1788 PredicateToUsers.erase(Result);
1792 // Mark users affected by a memory leader change.
1793 void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {
1794 for (auto M : CC->memory())
1795 markMemoryDefTouched(M);
1798 // Touch the instructions that need to be updated after a congruence class has a
1799 // leader change, and mark changed values.
1800 void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {
1801 for (auto M : *CC) {
1802 if (auto *I = dyn_cast<Instruction>(M))
1803 TouchedInstructions.set(InstrToDFSNum(I));
1804 LeaderChanges.insert(M);
1808 // Give a range of things that have instruction DFS numbers, this will return
1809 // the member of the range with the smallest dfs number.
1810 template <class T, class Range>
1811 T *NewGVN::getMinDFSOfRange(const Range &R) const {
1812 std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
1813 for (const auto X : R) {
1814 auto DFSNum = InstrToDFSNum(X);
1815 if (DFSNum < MinDFS.second)
1816 MinDFS = {X, DFSNum};
1818 return MinDFS.first;
1821 // This function returns the MemoryAccess that should be the next leader of
1822 // congruence class CC, under the assumption that the current leader is going to
1824 const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {
1825 // TODO: If this ends up to slow, we can maintain a next memory leader like we
1826 // do for regular leaders.
1827 // Make sure there will be a leader to find
1828 assert((CC->getStoreCount() > 0 || !CC->memory_empty()) &&
1829 "Can't get next leader if there is none");
1830 if (CC->getStoreCount() > 0) {
1831 if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first))
1832 return MSSA->getMemoryAccess(NL);
1833 // Find the store with the minimum DFS number.
1834 auto *V = getMinDFSOfRange<Value>(make_filter_range(
1835 *CC, [&](const Value *V) { return isa<StoreInst>(V); }));
1836 return MSSA->getMemoryAccess(cast<StoreInst>(V));
1838 assert(CC->getStoreCount() == 0);
1840 // Given our assertion, hitting this part must mean
1841 // !OldClass->memory_empty()
1842 if (CC->memory_size() == 1)
1843 return *CC->memory_begin();
1844 return getMinDFSOfRange<const MemoryPhi>(CC->memory());
1847 // This function returns the next value leader of a congruence class, under the
1848 // assumption that the current leader is going away. This should end up being
1849 // the next most dominating member.
1850 Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {
1851 // We don't need to sort members if there is only 1, and we don't care about
1852 // sorting the TOP class because everything either gets out of it or is
1855 if (CC->size() == 1 || CC == TOPClass) {
1856 return *(CC->begin());
1857 } else if (CC->getNextLeader().first) {
1858 ++NumGVNAvoidedSortedLeaderChanges;
1859 return CC->getNextLeader().first;
1861 ++NumGVNSortedLeaderChanges;
1862 // NOTE: If this ends up to slow, we can maintain a dual structure for
1863 // member testing/insertion, or keep things mostly sorted, and sort only
1864 // here, or use SparseBitVector or ....
1865 return getMinDFSOfRange<Value>(*CC);
1869 // Move a MemoryAccess, currently in OldClass, to NewClass, including updates to
1870 // the memory members, etc for the move.
1872 // The invariants of this function are:
1874 // I must be moving to NewClass from OldClass The StoreCount of OldClass and
1875 // NewClass is expected to have been updated for I already if it is is a store.
1876 // The OldClass memory leader has not been updated yet if I was the leader.
1877 void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,
1878 MemoryAccess *InstMA,
1879 CongruenceClass *OldClass,
1880 CongruenceClass *NewClass) {
1881 // If the leader is I, and we had a represenative MemoryAccess, it should
1882 // be the MemoryAccess of OldClass.
1883 assert((!InstMA || !OldClass->getMemoryLeader() ||
1884 OldClass->getLeader() != I ||
1885 OldClass->getMemoryLeader() == InstMA) &&
1886 "Representative MemoryAccess mismatch");
1887 // First, see what happens to the new class
1888 if (!NewClass->getMemoryLeader()) {
1889 // Should be a new class, or a store becoming a leader of a new class.
1890 assert(NewClass->size() == 1 ||
1891 (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));
1892 NewClass->setMemoryLeader(InstMA);
1893 // Mark it touched if we didn't just create a singleton
1894 DEBUG(dbgs() << "Memory class leader change for class " << NewClass->getID()
1895 << " due to new memory instruction becoming leader\n");
1896 markMemoryLeaderChangeTouched(NewClass);
1898 setMemoryClass(InstMA, NewClass);
1899 // Now, fixup the old class if necessary
1900 if (OldClass->getMemoryLeader() == InstMA) {
1901 if (OldClass->getStoreCount() != 0 || !OldClass->memory_empty()) {
1902 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1903 DEBUG(dbgs() << "Memory class leader change for class "
1904 << OldClass->getID() << " to "
1905 << *OldClass->getMemoryLeader()
1906 << " due to removal of old leader " << *InstMA << "\n");
1907 markMemoryLeaderChangeTouched(OldClass);
1909 OldClass->setMemoryLeader(nullptr);
1913 // Move a value, currently in OldClass, to be part of NewClass
1914 // Update OldClass and NewClass for the move (including changing leaders, etc).
1915 void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,
1916 CongruenceClass *OldClass,
1917 CongruenceClass *NewClass) {
1918 if (I == OldClass->getNextLeader().first)
1919 OldClass->resetNextLeader();
1921 // It's possible, though unlikely, for us to discover equivalences such
1922 // that the current leader does not dominate the old one.
1923 // This statistic tracks how often this happens.
1924 // We assert on phi nodes when this happens, currently, for debugging, because
1925 // we want to make sure we name phi node cycles properly.
1926 if (isa<Instruction>(NewClass->getLeader()) && NewClass->getLeader() &&
1927 I != NewClass->getLeader()) {
1928 auto *IBB = I->getParent();
1929 auto *NCBB = cast<Instruction>(NewClass->getLeader())->getParent();
1931 IBB == NCBB && InstrToDFSNum(I) < InstrToDFSNum(NewClass->getLeader());
1932 Dominated = Dominated || DT->properlyDominates(IBB, NCBB);
1934 ++NumGVNNotMostDominatingLeader;
1937 "New class for instruction should not be dominated by instruction");
1941 if (NewClass->getLeader() != I)
1942 NewClass->addPossibleNextLeader({I, InstrToDFSNum(I)});
1945 NewClass->insert(I);
1946 // Handle our special casing of stores.
1947 if (auto *SI = dyn_cast<StoreInst>(I)) {
1948 OldClass->decStoreCount();
1949 // Okay, so when do we want to make a store a leader of a class?
1950 // If we have a store defined by an earlier load, we want the earlier load
1951 // to lead the class.
1952 // If we have a store defined by something else, we want the store to lead
1953 // the class so everything else gets the "something else" as a value.
1954 // If we have a store as the single member of the class, we want the store
1956 if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) {
1957 // If it's a store expression we are using, it means we are not equivalent
1958 // to something earlier.
1959 if (isa<StoreExpression>(E)) {
1960 assert(lookupOperandLeader(SI->getValueOperand()) !=
1961 NewClass->getLeader());
1962 NewClass->setStoredValue(lookupOperandLeader(SI->getValueOperand()));
1963 markValueLeaderChangeTouched(NewClass);
1964 // Shift the new class leader to be the store
1965 DEBUG(dbgs() << "Changing leader of congruence class "
1966 << NewClass->getID() << " from " << *NewClass->getLeader()
1967 << " to " << *SI << " because store joined class\n");
1968 // If we changed the leader, we have to mark it changed because we don't
1969 // know what it will do to symbolic evlauation.
1970 NewClass->setLeader(SI);
1972 // We rely on the code below handling the MemoryAccess change.
1974 NewClass->incStoreCount();
1976 // True if there is no memory instructions left in a class that had memory
1977 // instructions before.
1979 // If it's not a memory use, set the MemoryAccess equivalence
1980 auto *InstMA = dyn_cast_or_null<MemoryDef>(MSSA->getMemoryAccess(I));
1981 bool InstWasMemoryLeader = InstMA && OldClass->getMemoryLeader() == InstMA;
1983 moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);
1984 ValueToClass[I] = NewClass;
1985 // See if we destroyed the class or need to swap leaders.
1986 if (OldClass->empty() && OldClass != TOPClass) {
1987 if (OldClass->getDefiningExpr()) {
1988 DEBUG(dbgs() << "Erasing expression " << OldClass->getDefiningExpr()
1989 << " from table\n");
1990 ExpressionToClass.erase(OldClass->getDefiningExpr());
1992 } else if (OldClass->getLeader() == I) {
1993 // When the leader changes, the value numbering of
1994 // everything may change due to symbolization changes, so we need to
1996 DEBUG(dbgs() << "Value class leader change for class " << OldClass->getID()
1998 ++NumGVNLeaderChanges;
1999 // Destroy the stored value if there are no more stores to represent it.
2000 // Note that this is basically clean up for the expression removal that
2001 // happens below. If we remove stores from a class, we may leave it as a
2002 // class of equivalent memory phis.
2003 if (OldClass->getStoreCount() == 0) {
2004 if (OldClass->getStoredValue())
2005 OldClass->setStoredValue(nullptr);
2007 // If we destroy the old access leader and it's a store, we have to
2008 // effectively destroy the congruence class. When it comes to scalars,
2009 // anything with the same value is as good as any other. That means that
2010 // one leader is as good as another, and as long as you have some leader for
2011 // the value, you are good.. When it comes to *memory states*, only one
2012 // particular thing really represents the definition of a given memory
2013 // state. Once it goes away, we need to re-evaluate which pieces of memory
2014 // are really still equivalent. The best way to do this is to re-value
2015 // number things. The only way to really make that happen is to destroy the
2016 // rest of the class. In order to effectively destroy the class, we reset
2017 // ExpressionToClass for each by using the ValueToExpression mapping. The
2018 // members later get marked as touched due to the leader change. We will
2019 // create new congruence classes, and the pieces that are still equivalent
2020 // will end back together in a new class. If this becomes too expensive, it
2021 // is possible to use a versioning scheme for the congruence classes to
2022 // avoid the expressions finding this old class. Note that the situation is
2023 // different for memory phis, becuase they are evaluated anew each time, and
2024 // they become equal not by hashing, but by seeing if all operands are the
2025 // same (or only one is reachable).
2026 if (OldClass->getStoreCount() > 0 && InstWasMemoryLeader) {
2027 DEBUG(dbgs() << "Kicking everything out of class " << OldClass->getID()
2028 << " because MemoryAccess leader changed");
2029 for (auto Member : *OldClass)
2030 ExpressionToClass.erase(ValueToExpression.lookup(Member));
2032 OldClass->setLeader(getNextValueLeader(OldClass));
2033 OldClass->resetNextLeader();
2034 markValueLeaderChangeTouched(OldClass);
2038 // Perform congruence finding on a given value numbering expression.
2039 void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
2040 ValueToExpression[I] = E;
2041 // This is guaranteed to return something, since it will at least find
2044 CongruenceClass *IClass = ValueToClass[I];
2045 assert(IClass && "Should have found a IClass");
2046 // Dead classes should have been eliminated from the mapping.
2047 assert(!IClass->isDead() && "Found a dead class");
2049 CongruenceClass *EClass;
2050 if (const auto *VE = dyn_cast<VariableExpression>(E)) {
2051 EClass = ValueToClass[VE->getVariableValue()];
2053 auto lookupResult = ExpressionToClass.insert({E, nullptr});
2055 // If it's not in the value table, create a new congruence class.
2056 if (lookupResult.second) {
2057 CongruenceClass *NewClass = createCongruenceClass(nullptr, E);
2058 auto place = lookupResult.first;
2059 place->second = NewClass;
2061 // Constants and variables should always be made the leader.
2062 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2063 NewClass->setLeader(CE->getConstantValue());
2064 } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {
2065 StoreInst *SI = SE->getStoreInst();
2066 NewClass->setLeader(SI);
2067 NewClass->setStoredValue(lookupOperandLeader(SI->getValueOperand()));
2068 // The RepMemoryAccess field will be filled in properly by the
2069 // moveValueToNewCongruenceClass call.
2071 NewClass->setLeader(I);
2073 assert(!isa<VariableExpression>(E) &&
2074 "VariableExpression should have been handled already");
2077 DEBUG(dbgs() << "Created new congruence class for " << *I
2078 << " using expression " << *E << " at " << NewClass->getID()
2079 << " and leader " << *(NewClass->getLeader()));
2080 if (NewClass->getStoredValue())
2081 DEBUG(dbgs() << " and stored value " << *(NewClass->getStoredValue()));
2082 DEBUG(dbgs() << "\n");
2084 EClass = lookupResult.first->second;
2085 if (isa<ConstantExpression>(E))
2086 assert((isa<Constant>(EClass->getLeader()) ||
2087 (EClass->getStoredValue() &&
2088 isa<Constant>(EClass->getStoredValue()))) &&
2089 "Any class with a constant expression should have a "
2092 assert(EClass && "Somehow don't have an eclass");
2094 assert(!EClass->isDead() && "We accidentally looked up a dead class");
2097 bool ClassChanged = IClass != EClass;
2098 bool LeaderChanged = LeaderChanges.erase(I);
2099 if (ClassChanged || LeaderChanged) {
2100 DEBUG(dbgs() << "New class " << EClass->getID() << " for expression " << *E
2103 moveValueToNewCongruenceClass(I, E, IClass, EClass);
2104 markUsersTouched(I);
2105 if (MemoryAccess *MA = MSSA->getMemoryAccess(I))
2106 markMemoryUsersTouched(MA);
2107 if (auto *CI = dyn_cast<CmpInst>(I))
2108 markPredicateUsersTouched(CI);
2112 // Process the fact that Edge (from, to) is reachable, including marking
2113 // any newly reachable blocks and instructions for processing.
2114 void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
2115 // Check if the Edge was reachable before.
2116 if (ReachableEdges.insert({From, To}).second) {
2117 // If this block wasn't reachable before, all instructions are touched.
2118 if (ReachableBlocks.insert(To).second) {
2119 DEBUG(dbgs() << "Block " << getBlockName(To) << " marked reachable\n");
2120 const auto &InstRange = BlockInstRange.lookup(To);
2121 TouchedInstructions.set(InstRange.first, InstRange.second);
2123 DEBUG(dbgs() << "Block " << getBlockName(To)
2124 << " was reachable, but new edge {" << getBlockName(From)
2125 << "," << getBlockName(To) << "} to it found\n");
2127 // We've made an edge reachable to an existing block, which may
2128 // impact predicates. Otherwise, only mark the phi nodes as touched, as
2129 // they are the only thing that depend on new edges. Anything using their
2130 // values will get propagated to if necessary.
2131 if (MemoryAccess *MemPhi = MSSA->getMemoryAccess(To))
2132 TouchedInstructions.set(InstrToDFSNum(MemPhi));
2134 auto BI = To->begin();
2135 while (isa<PHINode>(BI)) {
2136 TouchedInstructions.set(InstrToDFSNum(&*BI));
2143 // Given a predicate condition (from a switch, cmp, or whatever) and a block,
2144 // see if we know some constant value for it already.
2145 Value *NewGVN::findConditionEquivalence(Value *Cond) const {
2146 auto Result = lookupOperandLeader(Cond);
2147 if (isa<Constant>(Result))
2152 // Process the outgoing edges of a block for reachability.
2153 void NewGVN::processOutgoingEdges(TerminatorInst *TI, BasicBlock *B) {
2154 // Evaluate reachability of terminator instruction.
2156 if ((BR = dyn_cast<BranchInst>(TI)) && BR->isConditional()) {
2157 Value *Cond = BR->getCondition();
2158 Value *CondEvaluated = findConditionEquivalence(Cond);
2159 if (!CondEvaluated) {
2160 if (auto *I = dyn_cast<Instruction>(Cond)) {
2161 const Expression *E = createExpression(I);
2162 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2163 CondEvaluated = CE->getConstantValue();
2165 } else if (isa<ConstantInt>(Cond)) {
2166 CondEvaluated = Cond;
2170 BasicBlock *TrueSucc = BR->getSuccessor(0);
2171 BasicBlock *FalseSucc = BR->getSuccessor(1);
2172 if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) {
2174 DEBUG(dbgs() << "Condition for Terminator " << *TI
2175 << " evaluated to true\n");
2176 updateReachableEdge(B, TrueSucc);
2177 } else if (CI->isZero()) {
2178 DEBUG(dbgs() << "Condition for Terminator " << *TI
2179 << " evaluated to false\n");
2180 updateReachableEdge(B, FalseSucc);
2183 updateReachableEdge(B, TrueSucc);
2184 updateReachableEdge(B, FalseSucc);
2186 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
2187 // For switches, propagate the case values into the case
2190 // Remember how many outgoing edges there are to every successor.
2191 SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges;
2193 Value *SwitchCond = SI->getCondition();
2194 Value *CondEvaluated = findConditionEquivalence(SwitchCond);
2195 // See if we were able to turn this switch statement into a constant.
2196 if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) {
2197 auto *CondVal = cast<ConstantInt>(CondEvaluated);
2198 // We should be able to get case value for this.
2199 auto Case = *SI->findCaseValue(CondVal);
2200 if (Case.getCaseSuccessor() == SI->getDefaultDest()) {
2201 // We proved the value is outside of the range of the case.
2202 // We can't do anything other than mark the default dest as reachable,
2204 updateReachableEdge(B, SI->getDefaultDest());
2207 // Now get where it goes and mark it reachable.
2208 BasicBlock *TargetBlock = Case.getCaseSuccessor();
2209 updateReachableEdge(B, TargetBlock);
2211 for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; ++i) {
2212 BasicBlock *TargetBlock = SI->getSuccessor(i);
2213 ++SwitchEdges[TargetBlock];
2214 updateReachableEdge(B, TargetBlock);
2218 // Otherwise this is either unconditional, or a type we have no
2219 // idea about. Just mark successors as reachable.
2220 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
2221 BasicBlock *TargetBlock = TI->getSuccessor(i);
2222 updateReachableEdge(B, TargetBlock);
2225 // This also may be a memory defining terminator, in which case, set it
2226 // equivalent only to itself.
2228 auto *MA = MSSA->getMemoryAccess(TI);
2229 if (MA && !isa<MemoryUse>(MA)) {
2230 auto *CC = ensureLeaderOfMemoryClass(MA);
2231 if (setMemoryClass(MA, CC))
2232 markMemoryUsersTouched(MA);
2237 // The algorithm initially places the values of the routine in the TOP
2238 // congruence class. The leader of TOP is the undetermined value `undef`.
2239 // When the algorithm has finished, values still in TOP are unreachable.
2240 void NewGVN::initializeCongruenceClasses(Function &F) {
2241 NextCongruenceNum = 0;
2243 // Note that even though we use the live on entry def as a representative
2244 // MemoryAccess, it is *not* the same as the actual live on entry def. We
2245 // have no real equivalemnt to undef for MemoryAccesses, and so we really
2246 // should be checking whether the MemoryAccess is top if we want to know if it
2247 // is equivalent to everything. Otherwise, what this really signifies is that
2248 // the access "it reaches all the way back to the beginning of the function"
2250 // Initialize all other instructions to be in TOP class.
2251 TOPClass = createCongruenceClass(nullptr, nullptr);
2252 TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());
2253 // The live on entry def gets put into it's own class
2254 MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =
2255 createMemoryClass(MSSA->getLiveOnEntryDef());
2258 // All MemoryAccesses are equivalent to live on entry to start. They must
2259 // be initialized to something so that initial changes are noticed. For
2260 // the maximal answer, we initialize them all to be the same as
2262 auto *MemoryBlockDefs = MSSA->getBlockDefs(&B);
2263 if (MemoryBlockDefs)
2264 for (const auto &Def : *MemoryBlockDefs) {
2265 MemoryAccessToClass[&Def] = TOPClass;
2266 auto *MD = dyn_cast<MemoryDef>(&Def);
2267 // Insert the memory phis into the member list.
2269 const MemoryPhi *MP = cast<MemoryPhi>(&Def);
2270 TOPClass->memory_insert(MP);
2271 MemoryPhiState.insert({MP, MPS_TOP});
2274 if (MD && isa<StoreInst>(MD->getMemoryInst()))
2275 TOPClass->incStoreCount();
2278 // Don't insert void terminators into the class. We don't value number
2279 // them, and they just end up sitting in TOP.
2280 if (isa<TerminatorInst>(I) && I.getType()->isVoidTy())
2282 TOPClass->insert(&I);
2283 ValueToClass[&I] = TOPClass;
2287 // Initialize arguments to be in their own unique congruence classes
2288 for (auto &FA : F.args())
2289 createSingletonCongruenceClass(&FA);
2292 void NewGVN::cleanupTables() {
2293 for (unsigned i = 0, e = CongruenceClasses.size(); i != e; ++i) {
2294 DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->getID()
2295 << " has " << CongruenceClasses[i]->size() << " members\n");
2296 // Make sure we delete the congruence class (probably worth switching to
2297 // a unique_ptr at some point.
2298 delete CongruenceClasses[i];
2299 CongruenceClasses[i] = nullptr;
2302 ValueToClass.clear();
2303 ArgRecycler.clear(ExpressionAllocator);
2304 ExpressionAllocator.Reset();
2305 CongruenceClasses.clear();
2306 ExpressionToClass.clear();
2307 ValueToExpression.clear();
2308 ReachableBlocks.clear();
2309 ReachableEdges.clear();
2311 ProcessedCount.clear();
2314 InstructionsToErase.clear();
2316 BlockInstRange.clear();
2317 TouchedInstructions.clear();
2318 MemoryAccessToClass.clear();
2319 PredicateToUsers.clear();
2320 MemoryToUsers.clear();
2323 std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
2325 unsigned End = Start;
2326 if (MemoryAccess *MemPhi = MSSA->getMemoryAccess(B)) {
2327 InstrDFS[MemPhi] = End++;
2328 DFSToInstr.emplace_back(MemPhi);
2331 for (auto &I : *B) {
2332 // There's no need to call isInstructionTriviallyDead more than once on
2333 // an instruction. Therefore, once we know that an instruction is dead
2334 // we change its DFS number so that it doesn't get value numbered.
2335 if (isInstructionTriviallyDead(&I, TLI)) {
2337 DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");
2338 markInstructionForDeletion(&I);
2342 InstrDFS[&I] = End++;
2343 DFSToInstr.emplace_back(&I);
2346 // All of the range functions taken half-open ranges (open on the end side).
2347 // So we do not subtract one from count, because at this point it is one
2348 // greater than the last instruction.
2349 return std::make_pair(Start, End);
2352 void NewGVN::updateProcessedCount(Value *V) {
2354 if (ProcessedCount.count(V) == 0) {
2355 ProcessedCount.insert({V, 1});
2357 ++ProcessedCount[V];
2358 assert(ProcessedCount[V] < 100 &&
2359 "Seem to have processed the same Value a lot");
2363 // Evaluate MemoryPhi nodes symbolically, just like PHI nodes
2364 void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
2365 // If all the arguments are the same, the MemoryPhi has the same value as the
2367 // Filter out unreachable blocks and self phis from our operands.
2368 const BasicBlock *PHIBlock = MP->getBlock();
2369 auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {
2370 return lookupMemoryLeader(cast<MemoryAccess>(U)) != MP &&
2371 !isMemoryAccessTop(cast<MemoryAccess>(U)) &&
2372 ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock});
2374 // If all that is left is nothing, our memoryphi is undef. We keep it as
2375 // InitialClass. Note: The only case this should happen is if we have at
2376 // least one self-argument.
2377 if (Filtered.begin() == Filtered.end()) {
2378 if (setMemoryClass(MP, TOPClass))
2379 markMemoryUsersTouched(MP);
2383 // Transform the remaining operands into operand leaders.
2384 // FIXME: mapped_iterator should have a range version.
2385 auto LookupFunc = [&](const Use &U) {
2386 return lookupMemoryLeader(cast<MemoryAccess>(U));
2388 auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);
2389 auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);
2391 // and now check if all the elements are equal.
2392 // Sadly, we can't use std::equals since these are random access iterators.
2393 const auto *AllSameValue = *MappedBegin;
2395 bool AllEqual = std::all_of(
2396 MappedBegin, MappedEnd,
2397 [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });
2400 DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue << "\n");
2402 DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
2403 // If it's equal to something, it's in that class. Otherwise, it has to be in
2404 // a class where it is the leader (other things may be equivalent to it, but
2405 // it needs to start off in its own class, which means it must have been the
2406 // leader, and it can't have stopped being the leader because it was never
2408 CongruenceClass *CC =
2409 AllEqual ? getMemoryClass(AllSameValue) : ensureLeaderOfMemoryClass(MP);
2410 auto OldState = MemoryPhiState.lookup(MP);
2411 assert(OldState != MPS_Invalid && "Invalid memory phi state");
2412 auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique;
2413 MemoryPhiState[MP] = NewState;
2414 if (setMemoryClass(MP, CC) || OldState != NewState)
2415 markMemoryUsersTouched(MP);
2418 // Value number a single instruction, symbolically evaluating, performing
2419 // congruence finding, and updating mappings.
2420 void NewGVN::valueNumberInstruction(Instruction *I) {
2421 DEBUG(dbgs() << "Processing instruction " << *I << "\n");
2422 if (!I->isTerminator()) {
2423 const Expression *Symbolized = nullptr;
2424 if (DebugCounter::shouldExecute(VNCounter)) {
2425 Symbolized = performSymbolicEvaluation(I);
2427 // Mark the instruction as unused so we don't value number it again.
2430 // If we couldn't come up with a symbolic expression, use the unknown
2432 if (Symbolized == nullptr) {
2433 Symbolized = createUnknownExpression(I);
2436 performCongruenceFinding(I, Symbolized);
2438 // Handle terminators that return values. All of them produce values we
2439 // don't currently understand. We don't place non-value producing
2440 // terminators in a class.
2441 if (!I->getType()->isVoidTy()) {
2442 auto *Symbolized = createUnknownExpression(I);
2443 performCongruenceFinding(I, Symbolized);
2445 processOutgoingEdges(dyn_cast<TerminatorInst>(I), I->getParent());
2449 // Check if there is a path, using single or equal argument phi nodes, from
2451 bool NewGVN::singleReachablePHIPath(const MemoryAccess *First,
2452 const MemoryAccess *Second) const {
2453 if (First == Second)
2455 if (MSSA->isLiveOnEntryDef(First))
2458 const auto *EndDef = First;
2459 for (auto *ChainDef : optimized_def_chain(First)) {
2460 if (ChainDef == Second)
2462 if (MSSA->isLiveOnEntryDef(ChainDef))
2466 auto *MP = cast<MemoryPhi>(EndDef);
2467 auto ReachableOperandPred = [&](const Use &U) {
2468 return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()});
2470 auto FilteredPhiArgs =
2471 make_filter_range(MP->operands(), ReachableOperandPred);
2472 SmallVector<const Value *, 32> OperandList;
2473 std::copy(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
2474 std::back_inserter(OperandList));
2475 bool Okay = OperandList.size() == 1;
2478 std::equal(OperandList.begin(), OperandList.end(), OperandList.begin());
2480 return singleReachablePHIPath(cast<MemoryAccess>(OperandList[0]), Second);
2484 // Verify the that the memory equivalence table makes sense relative to the
2485 // congruence classes. Note that this checking is not perfect, and is currently
2486 // subject to very rare false negatives. It is only useful for
2487 // testing/debugging.
2488 void NewGVN::verifyMemoryCongruency() const {
2490 // Verify that the memory table equivalence and memory member set match
2491 for (const auto *CC : CongruenceClasses) {
2492 if (CC == TOPClass || CC->isDead())
2494 if (CC->getStoreCount() != 0) {
2495 assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&
2496 "Any class with a store as a "
2497 "leader should have a "
2498 "representative stored value\n");
2499 assert(CC->getMemoryLeader() &&
2500 "Any congruence class with a store should "
2501 "have a representative access\n");
2504 if (CC->getMemoryLeader())
2505 assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&
2506 "Representative MemoryAccess does not appear to be reverse "
2508 for (auto M : CC->memory())
2509 assert(MemoryAccessToClass.lookup(M) == CC &&
2510 "Memory member does not appear to be reverse mapped properly");
2513 // Anything equivalent in the MemoryAccess table should be in the same
2514 // congruence class.
2516 // Filter out the unreachable and trivially dead entries, because they may
2517 // never have been updated if the instructions were not processed.
2518 auto ReachableAccessPred =
2519 [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {
2520 bool Result = ReachableBlocks.count(Pair.first->getBlock());
2523 if (MSSA->isLiveOnEntryDef(Pair.first))
2525 if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
2526 return !isInstructionTriviallyDead(MemDef->getMemoryInst());
2527 if (MemoryToDFSNum(Pair.first) == 0)
2532 auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);
2533 for (auto KV : Filtered) {
2534 assert(KV.second != TOPClass &&
2535 "Memory not unreachable but ended up in TOP");
2536 if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
2537 auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());
2538 if (FirstMUD && SecondMUD)
2539 assert((singleReachablePHIPath(FirstMUD, SecondMUD) ||
2540 ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
2541 ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
2542 "The instructions for these memory operations should have "
2543 "been in the same congruence class or reachable through"
2544 "a single argument phi");
2545 } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
2546 // We can only sanely verify that MemoryDefs in the operand list all have
2548 auto ReachableOperandPred = [&](const Use &U) {
2549 return ReachableEdges.count(
2550 {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&
2554 // All arguments should in the same class, ignoring unreachable arguments
2555 auto FilteredPhiArgs =
2556 make_filter_range(FirstMP->operands(), ReachableOperandPred);
2557 SmallVector<const CongruenceClass *, 16> PhiOpClasses;
2558 std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
2559 std::back_inserter(PhiOpClasses), [&](const Use &U) {
2560 const MemoryDef *MD = cast<MemoryDef>(U);
2561 return ValueToClass.lookup(MD->getMemoryInst());
2563 assert(std::equal(PhiOpClasses.begin(), PhiOpClasses.end(),
2564 PhiOpClasses.begin()) &&
2565 "All MemoryPhi arguments should be in the same class");
2571 // Verify that the sparse propagation we did actually found the maximal fixpoint
2572 // We do this by storing the value to class mapping, touching all instructions,
2573 // and redoing the iteration to see if anything changed.
2574 void NewGVN::verifyIterationSettled(Function &F) {
2576 DEBUG(dbgs() << "Beginning iteration verification\n");
2577 if (DebugCounter::isCounterSet(VNCounter))
2578 DebugCounter::setCounterValue(VNCounter, StartingVNCounter);
2580 // Note that we have to store the actual classes, as we may change existing
2581 // classes during iteration. This is because our memory iteration propagation
2582 // is not perfect, and so may waste a little work. But it should generate
2583 // exactly the same congruence classes we have now, with different IDs.
2584 std::map<const Value *, CongruenceClass> BeforeIteration;
2586 for (auto &KV : ValueToClass) {
2587 if (auto *I = dyn_cast<Instruction>(KV.first))
2588 // Skip unused/dead instructions.
2589 if (InstrToDFSNum(I) == 0)
2591 BeforeIteration.insert({KV.first, *KV.second});
2594 TouchedInstructions.set();
2595 TouchedInstructions.reset(0);
2596 iterateTouchedInstructions();
2597 DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>>
2599 for (const auto &KV : ValueToClass) {
2600 if (auto *I = dyn_cast<Instruction>(KV.first))
2601 // Skip unused/dead instructions.
2602 if (InstrToDFSNum(I) == 0)
2604 // We could sink these uses, but i think this adds a bit of clarity here as
2605 // to what we are comparing.
2606 auto *BeforeCC = &BeforeIteration.find(KV.first)->second;
2607 auto *AfterCC = KV.second;
2608 // Note that the classes can't change at this point, so we memoize the set
2610 if (!EqualClasses.count({BeforeCC, AfterCC})) {
2611 assert(BeforeCC->isEquivalentTo(AfterCC) &&
2612 "Value number changed after main loop completed!");
2613 EqualClasses.insert({BeforeCC, AfterCC});
2619 // This is the main value numbering loop, it iterates over the initial touched
2620 // instruction set, propagating value numbers, marking things touched, etc,
2621 // until the set of touched instructions is completely empty.
2622 void NewGVN::iterateTouchedInstructions() {
2623 unsigned int Iterations = 0;
2624 // Figure out where touchedinstructions starts
2625 int FirstInstr = TouchedInstructions.find_first();
2626 // Nothing set, nothing to iterate, just return.
2627 if (FirstInstr == -1)
2629 BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr));
2630 while (TouchedInstructions.any()) {
2632 // Walk through all the instructions in all the blocks in RPO.
2633 // TODO: As we hit a new block, we should push and pop equalities into a
2634 // table lookupOperandLeader can use, to catch things PredicateInfo
2635 // might miss, like edge-only equivalences.
2636 for (int InstrNum = TouchedInstructions.find_first(); InstrNum != -1;
2637 InstrNum = TouchedInstructions.find_next(InstrNum)) {
2639 // This instruction was found to be dead. We don't bother looking
2641 if (InstrNum == 0) {
2642 TouchedInstructions.reset(InstrNum);
2646 Value *V = InstrFromDFSNum(InstrNum);
2647 BasicBlock *CurrBlock = getBlockForValue(V);
2649 // If we hit a new block, do reachability processing.
2650 if (CurrBlock != LastBlock) {
2651 LastBlock = CurrBlock;
2652 bool BlockReachable = ReachableBlocks.count(CurrBlock);
2653 const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);
2655 // If it's not reachable, erase any touched instructions and move on.
2656 if (!BlockReachable) {
2657 TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);
2658 DEBUG(dbgs() << "Skipping instructions in block "
2659 << getBlockName(CurrBlock)
2660 << " because it is unreachable\n");
2663 updateProcessedCount(CurrBlock);
2666 if (auto *MP = dyn_cast<MemoryPhi>(V)) {
2667 DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
2668 valueNumberMemoryPhi(MP);
2669 } else if (auto *I = dyn_cast<Instruction>(V)) {
2670 valueNumberInstruction(I);
2672 llvm_unreachable("Should have been a MemoryPhi or Instruction");
2674 updateProcessedCount(V);
2675 // Reset after processing (because we may mark ourselves as touched when
2676 // we propagate equalities).
2677 TouchedInstructions.reset(InstrNum);
2680 NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);
2683 // This is the main transformation entry point.
2684 bool NewGVN::runGVN() {
2685 if (DebugCounter::isCounterSet(VNCounter))
2686 StartingVNCounter = DebugCounter::getCounterValue(VNCounter);
2687 bool Changed = false;
2688 NumFuncArgs = F.arg_size();
2689 MSSAWalker = MSSA->getWalker();
2691 // Count number of instructions for sizing of hash tables, and come
2692 // up with a global dfs numbering for instructions.
2693 unsigned ICount = 1;
2694 // Add an empty instruction to account for the fact that we start at 1
2695 DFSToInstr.emplace_back(nullptr);
2696 // Note: We want ideal RPO traversal of the blocks, which is not quite the
2697 // same as dominator tree order, particularly with regard whether backedges
2698 // get visited first or second, given a block with multiple successors.
2699 // If we visit in the wrong order, we will end up performing N times as many
2701 // The dominator tree does guarantee that, for a given dom tree node, it's
2702 // parent must occur before it in the RPO ordering. Thus, we only need to sort
2704 ReversePostOrderTraversal<Function *> RPOT(&F);
2705 unsigned Counter = 0;
2706 for (auto &B : RPOT) {
2707 auto *Node = DT->getNode(B);
2708 assert(Node && "RPO and Dominator tree should have same reachability");
2709 RPOOrdering[Node] = ++Counter;
2711 // Sort dominator tree children arrays into RPO.
2712 for (auto &B : RPOT) {
2713 auto *Node = DT->getNode(B);
2714 if (Node->getChildren().size() > 1)
2715 std::sort(Node->begin(), Node->end(),
2716 [&](const DomTreeNode *A, const DomTreeNode *B) {
2717 return RPOOrdering[A] < RPOOrdering[B];
2721 // Now a standard depth first ordering of the domtree is equivalent to RPO.
2722 auto DFI = df_begin(DT->getRootNode());
2723 for (auto DFE = df_end(DT->getRootNode()); DFI != DFE; ++DFI) {
2724 BasicBlock *B = DFI->getBlock();
2725 const auto &BlockRange = assignDFSNumbers(B, ICount);
2726 BlockInstRange.insert({B, BlockRange});
2727 ICount += BlockRange.second - BlockRange.first;
2730 // Handle forward unreachable blocks and figure out which blocks
2731 // have single preds.
2733 // Assign numbers to unreachable blocks.
2734 if (!DFI.nodeVisited(DT->getNode(&B))) {
2735 const auto &BlockRange = assignDFSNumbers(&B, ICount);
2736 BlockInstRange.insert({&B, BlockRange});
2737 ICount += BlockRange.second - BlockRange.first;
2741 TouchedInstructions.resize(ICount);
2742 // Ensure we don't end up resizing the expressionToClass map, as
2743 // that can be quite expensive. At most, we have one expression per
2745 ExpressionToClass.reserve(ICount);
2747 // Initialize the touched instructions to include the entry block.
2748 const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());
2749 TouchedInstructions.set(InstRange.first, InstRange.second);
2750 ReachableBlocks.insert(&F.getEntryBlock());
2752 initializeCongruenceClasses(F);
2753 iterateTouchedInstructions();
2754 verifyMemoryCongruency();
2755 verifyIterationSettled(F);
2757 Changed |= eliminateInstructions(F);
2759 // Delete all instructions marked for deletion.
2760 for (Instruction *ToErase : InstructionsToErase) {
2761 if (!ToErase->use_empty())
2762 ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType()));
2764 ToErase->eraseFromParent();
2767 // Delete all unreachable blocks.
2768 auto UnreachableBlockPred = [&](const BasicBlock &BB) {
2769 return !ReachableBlocks.count(&BB);
2772 for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {
2773 DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
2774 << " is unreachable\n");
2775 deleteInstructionsInBlock(&BB);
2783 // Return true if V is a value that will always be available (IE can
2784 // be placed anywhere) in the function. We don't do globals here
2785 // because they are often worse to put in place.
2786 // TODO: Separate cost from availability
2787 static bool alwaysAvailable(Value *V) {
2788 return isa<Constant>(V) || isa<Argument>(V);
2791 struct NewGVN::ValueDFS {
2795 // Only one of Def and U will be set.
2796 // The bool in the Def tells us whether the Def is the stored value of a
2798 PointerIntPair<Value *, 1, bool> Def;
2800 bool operator<(const ValueDFS &Other) const {
2801 // It's not enough that any given field be less than - we have sets
2802 // of fields that need to be evaluated together to give a proper ordering.
2803 // For example, if you have;
2808 // We want the second to be less than the first, but if we just go field
2809 // by field, we will get to Val 0 < Val 50 and say the first is less than
2810 // the second. We only want it to be less than if the DFS orders are equal.
2812 // Each LLVM instruction only produces one value, and thus the lowest-level
2813 // differentiator that really matters for the stack (and what we use as as a
2814 // replacement) is the local dfs number.
2815 // Everything else in the structure is instruction level, and only affects
2816 // the order in which we will replace operands of a given instruction.
2818 // For a given instruction (IE things with equal dfsin, dfsout, localnum),
2819 // the order of replacement of uses does not matter.
2823 // When you hit b, you will have two valuedfs with the same dfsin, out, and
2825 // The .val will be the same as well.
2826 // The .u's will be different.
2827 // You will replace both, and it does not matter what order you replace them
2828 // in (IE whether you replace operand 2, then operand 1, or operand 1, then
2830 // Similarly for the case of same dfsin, dfsout, localnum, but different
2835 // in c, we will a valuedfs for a, and one for b,with everything the same
2837 // It does not matter what order we replace these operands in.
2838 // You will always end up with the same IR, and this is guaranteed.
2839 return std::tie(DFSIn, DFSOut, LocalNum, Def, U) <
2840 std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def,
2845 // This function converts the set of members for a congruence class from values,
2846 // to sets of defs and uses with associated DFS info. The total number of
2847 // reachable uses for each value is stored in UseCount, and instructions that
2849 // dead (have no non-dead uses) are stored in ProbablyDead.
2850 void NewGVN::convertClassToDFSOrdered(
2851 const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,
2852 DenseMap<const Value *, unsigned int> &UseCounts,
2853 SmallPtrSetImpl<Instruction *> &ProbablyDead) const {
2854 for (auto D : Dense) {
2855 // First add the value.
2856 BasicBlock *BB = getBlockForValue(D);
2857 // Constants are handled prior to ever calling this function, so
2858 // we should only be left with instructions as members.
2859 assert(BB && "Should have figured out a basic block for value");
2861 DomTreeNode *DomNode = DT->getNode(BB);
2862 VDDef.DFSIn = DomNode->getDFSNumIn();
2863 VDDef.DFSOut = DomNode->getDFSNumOut();
2864 // If it's a store, use the leader of the value operand, if it's always
2865 // available, or the value operand. TODO: We could do dominance checks to
2866 // find a dominating leader, but not worth it ATM.
2867 if (auto *SI = dyn_cast<StoreInst>(D)) {
2868 auto Leader = lookupOperandLeader(SI->getValueOperand());
2869 if (alwaysAvailable(Leader)) {
2870 VDDef.Def.setPointer(Leader);
2872 VDDef.Def.setPointer(SI->getValueOperand());
2873 VDDef.Def.setInt(true);
2876 VDDef.Def.setPointer(D);
2878 assert(isa<Instruction>(D) &&
2879 "The dense set member should always be an instruction");
2880 VDDef.LocalNum = InstrToDFSNum(D);
2881 DFSOrderedSet.emplace_back(VDDef);
2882 Instruction *Def = cast<Instruction>(D);
2883 unsigned int UseCount = 0;
2884 // Now add the uses.
2885 for (auto &U : Def->uses()) {
2886 if (auto *I = dyn_cast<Instruction>(U.getUser())) {
2887 // Don't try to replace into dead uses
2888 if (InstructionsToErase.count(I))
2891 // Put the phi node uses in the incoming block.
2893 if (auto *P = dyn_cast<PHINode>(I)) {
2894 IBlock = P->getIncomingBlock(U);
2895 // Make phi node users appear last in the incoming block
2897 VDUse.LocalNum = InstrDFS.size() + 1;
2899 IBlock = I->getParent();
2900 VDUse.LocalNum = InstrToDFSNum(I);
2903 // Skip uses in unreachable blocks, as we're going
2905 if (ReachableBlocks.count(IBlock) == 0)
2908 DomTreeNode *DomNode = DT->getNode(IBlock);
2909 VDUse.DFSIn = DomNode->getDFSNumIn();
2910 VDUse.DFSOut = DomNode->getDFSNumOut();
2913 DFSOrderedSet.emplace_back(VDUse);
2917 // If there are no uses, it's probably dead (but it may have side-effects,
2918 // so not definitely dead. Otherwise, store the number of uses so we can
2919 // track if it becomes dead later).
2921 ProbablyDead.insert(Def);
2923 UseCounts[Def] = UseCount;
2927 // This function converts the set of members for a congruence class from values,
2928 // to the set of defs for loads and stores, with associated DFS info.
2929 void NewGVN::convertClassToLoadsAndStores(
2930 const CongruenceClass &Dense,
2931 SmallVectorImpl<ValueDFS> &LoadsAndStores) const {
2932 for (auto D : Dense) {
2933 if (!isa<LoadInst>(D) && !isa<StoreInst>(D))
2936 BasicBlock *BB = getBlockForValue(D);
2938 DomTreeNode *DomNode = DT->getNode(BB);
2939 VD.DFSIn = DomNode->getDFSNumIn();
2940 VD.DFSOut = DomNode->getDFSNumOut();
2941 VD.Def.setPointer(D);
2943 // If it's an instruction, use the real local dfs number.
2944 if (auto *I = dyn_cast<Instruction>(D))
2945 VD.LocalNum = InstrToDFSNum(I);
2947 llvm_unreachable("Should have been an instruction");
2949 LoadsAndStores.emplace_back(VD);
2953 static void patchReplacementInstruction(Instruction *I, Value *Repl) {
2954 auto *ReplInst = dyn_cast<Instruction>(Repl);
2958 // Patch the replacement so that it is not more restrictive than the value
2960 // Note that if 'I' is a load being replaced by some operation,
2961 // for example, by an arithmetic operation, then andIRFlags()
2962 // would just erase all math flags from the original arithmetic
2963 // operation, which is clearly not wanted and not needed.
2964 if (!isa<LoadInst>(I))
2965 ReplInst->andIRFlags(I);
2967 // FIXME: If both the original and replacement value are part of the
2968 // same control-flow region (meaning that the execution of one
2969 // guarantees the execution of the other), then we can combine the
2970 // noalias scopes here and do better than the general conservative
2971 // answer used in combineMetadata().
2973 // In general, GVN unifies expressions over different control-flow
2974 // regions, and so we need a conservative combination of the noalias
2976 static const unsigned KnownIDs[] = {
2977 LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
2978 LLVMContext::MD_noalias, LLVMContext::MD_range,
2979 LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load,
2980 LLVMContext::MD_invariant_group};
2981 combineMetadata(ReplInst, I, KnownIDs);
2984 static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
2985 patchReplacementInstruction(I, Repl);
2986 I->replaceAllUsesWith(Repl);
2989 void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
2990 DEBUG(dbgs() << " BasicBlock Dead:" << *BB);
2991 ++NumGVNBlocksDeleted;
2993 // Delete the instructions backwards, as it has a reduced likelihood of having
2994 // to update as many def-use and use-def chains. Start after the terminator.
2995 auto StartPoint = BB->rbegin();
2997 // Note that we explicitly recalculate BB->rend() on each iteration,
2998 // as it may change when we remove the first instruction.
2999 for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
3000 Instruction &Inst = *I++;
3001 if (!Inst.use_empty())
3002 Inst.replaceAllUsesWith(UndefValue::get(Inst.getType()));
3003 if (isa<LandingPadInst>(Inst))
3006 Inst.eraseFromParent();
3007 ++NumGVNInstrDeleted;
3009 // Now insert something that simplifycfg will turn into an unreachable.
3010 Type *Int8Ty = Type::getInt8Ty(BB->getContext());
3011 new StoreInst(UndefValue::get(Int8Ty),
3012 Constant::getNullValue(Int8Ty->getPointerTo()),
3013 BB->getTerminator());
3016 void NewGVN::markInstructionForDeletion(Instruction *I) {
3017 DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
3018 InstructionsToErase.insert(I);
3021 void NewGVN::replaceInstruction(Instruction *I, Value *V) {
3023 DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
3024 patchAndReplaceAllUsesWith(I, V);
3025 // We save the actual erasing to avoid invalidating memory
3026 // dependencies until we are done with everything.
3027 markInstructionForDeletion(I);
3032 // This is a stack that contains both the value and dfs info of where
3033 // that value is valid.
3034 class ValueDFSStack {
3036 Value *back() const { return ValueStack.back(); }
3037 std::pair<int, int> dfs_back() const { return DFSStack.back(); }
3039 void push_back(Value *V, int DFSIn, int DFSOut) {
3040 ValueStack.emplace_back(V);
3041 DFSStack.emplace_back(DFSIn, DFSOut);
3043 bool empty() const { return DFSStack.empty(); }
3044 bool isInScope(int DFSIn, int DFSOut) const {
3047 return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
3050 void popUntilDFSScope(int DFSIn, int DFSOut) {
3052 // These two should always be in sync at this point.
3053 assert(ValueStack.size() == DFSStack.size() &&
3054 "Mismatch between ValueStack and DFSStack");
3056 !DFSStack.empty() &&
3057 !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {
3058 DFSStack.pop_back();
3059 ValueStack.pop_back();
3064 SmallVector<Value *, 8> ValueStack;
3065 SmallVector<std::pair<int, int>, 8> DFSStack;
3069 bool NewGVN::eliminateInstructions(Function &F) {
3070 // This is a non-standard eliminator. The normal way to eliminate is
3071 // to walk the dominator tree in order, keeping track of available
3072 // values, and eliminating them. However, this is mildly
3073 // pointless. It requires doing lookups on every instruction,
3074 // regardless of whether we will ever eliminate it. For
3075 // instructions part of most singleton congruence classes, we know we
3076 // will never eliminate them.
3078 // Instead, this eliminator looks at the congruence classes directly, sorts
3079 // them into a DFS ordering of the dominator tree, and then we just
3080 // perform elimination straight on the sets by walking the congruence
3081 // class member uses in order, and eliminate the ones dominated by the
3082 // last member. This is worst case O(E log E) where E = number of
3083 // instructions in a single congruence class. In theory, this is all
3084 // instructions. In practice, it is much faster, as most instructions are
3085 // either in singleton congruence classes or can't possibly be eliminated
3086 // anyway (if there are no overlapping DFS ranges in class).
3087 // When we find something not dominated, it becomes the new leader
3088 // for elimination purposes.
3089 // TODO: If we wanted to be faster, We could remove any members with no
3090 // overlapping ranges while sorting, as we will never eliminate anything
3091 // with those members, as they don't dominate anything else in our set.
3093 bool AnythingReplaced = false;
3095 // Since we are going to walk the domtree anyway, and we can't guarantee the
3096 // DFS numbers are updated, we compute some ourselves.
3097 DT->updateDFSNumbers();
3100 if (!ReachableBlocks.count(&B)) {
3101 for (const auto S : successors(&B)) {
3102 for (auto II = S->begin(); isa<PHINode>(II); ++II) {
3103 auto &Phi = cast<PHINode>(*II);
3104 DEBUG(dbgs() << "Replacing incoming value of " << *II << " for block "
3106 << " with undef due to it being unreachable\n");
3107 for (auto &Operand : Phi.incoming_values())
3108 if (Phi.getIncomingBlock(Operand) == &B)
3109 Operand.set(UndefValue::get(Phi.getType()));
3115 // Map to store the use counts
3116 DenseMap<const Value *, unsigned int> UseCounts;
3117 for (CongruenceClass *CC : reverse(CongruenceClasses)) {
3118 // Track the equivalent store info so we can decide whether to try
3119 // dead store elimination.
3120 SmallVector<ValueDFS, 8> PossibleDeadStores;
3121 SmallPtrSet<Instruction *, 8> ProbablyDead;
3122 if (CC->isDead() || CC->empty())
3124 // Everything still in the TOP class is unreachable or dead.
3125 if (CC == TOPClass) {
3128 assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||
3129 InstructionsToErase.count(cast<Instruction>(M))) &&
3130 "Everything in TOP should be unreachable or dead at this "
3136 assert(CC->getLeader() && "We should have had a leader");
3137 // If this is a leader that is always available, and it's a
3138 // constant or has no equivalences, just replace everything with
3139 // it. We then update the congruence class with whatever members
3142 CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
3143 if (alwaysAvailable(Leader)) {
3144 CongruenceClass::MemberSet MembersLeft;
3145 for (auto M : *CC) {
3147 // Void things have no uses we can replace.
3148 if (Member == Leader || !isa<Instruction>(Member) ||
3149 Member->getType()->isVoidTy()) {
3150 MembersLeft.insert(Member);
3153 DEBUG(dbgs() << "Found replacement " << *(Leader) << " for " << *Member
3155 auto *I = cast<Instruction>(Member);
3156 assert(Leader != I && "About to accidentally remove our leader");
3157 replaceInstruction(I, Leader);
3158 AnythingReplaced = true;
3160 CC->swap(MembersLeft);
3162 DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID()
3164 // If this is a singleton, we can skip it.
3165 if (CC->size() != 1) {
3166 // This is a stack because equality replacement/etc may place
3167 // constants in the middle of the member list, and we want to use
3168 // those constant values in preference to the current leader, over
3169 // the scope of those constants.
3170 ValueDFSStack EliminationStack;
3172 // Convert the members to DFS ordered sets and then merge them.
3173 SmallVector<ValueDFS, 8> DFSOrderedSet;
3174 convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead);
3176 // Sort the whole thing.
3177 std::sort(DFSOrderedSet.begin(), DFSOrderedSet.end());
3178 for (auto &VD : DFSOrderedSet) {
3179 int MemberDFSIn = VD.DFSIn;
3180 int MemberDFSOut = VD.DFSOut;
3181 Value *Def = VD.Def.getPointer();
3182 bool FromStore = VD.Def.getInt();
3184 // We ignore void things because we can't get a value from them.
3185 if (Def && Def->getType()->isVoidTy())
3188 if (EliminationStack.empty()) {
3189 DEBUG(dbgs() << "Elimination Stack is empty\n");
3191 DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
3192 << EliminationStack.dfs_back().first << ","
3193 << EliminationStack.dfs_back().second << ")\n");
3196 DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
3197 << MemberDFSOut << ")\n");
3198 // First, we see if we are out of scope or empty. If so,
3199 // and there equivalences, we try to replace the top of
3200 // stack with equivalences (if it's on the stack, it must
3201 // not have been eliminated yet).
3202 // Then we synchronize to our current scope, by
3203 // popping until we are back within a DFS scope that
3204 // dominates the current member.
3205 // Then, what happens depends on a few factors
3206 // If the stack is now empty, we need to push
3207 // If we have a constant or a local equivalence we want to
3208 // start using, we also push.
3209 // Otherwise, we walk along, processing members who are
3210 // dominated by this scope, and eliminate them.
3211 bool ShouldPush = Def && EliminationStack.empty();
3213 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);
3215 if (OutOfScope || ShouldPush) {
3216 // Sync to our current scope.
3217 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
3218 bool ShouldPush = Def && EliminationStack.empty();
3220 EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut);
3224 // Skip the Def's, we only want to eliminate on their uses. But mark
3225 // dominated defs as dead.
3227 // For anything in this case, what and how we value number
3228 // guarantees that any side-effets that would have occurred (ie
3229 // throwing, etc) can be proven to either still occur (because it's
3230 // dominated by something that has the same side-effects), or never
3231 // occur. Otherwise, we would not have been able to prove it value
3232 // equivalent to something else. For these things, we can just mark
3233 // it all dead. Note that this is different from the "ProbablyDead"
3234 // set, which may not be dominated by anything, and thus, are only
3235 // easy to prove dead if they are also side-effect free. Note that
3236 // because stores are put in terms of the stored value, we skip
3237 // stored values here. If the stored value is really dead, it will
3238 // still be marked for deletion when we process it in its own class.
3239 if (!EliminationStack.empty() && Def != EliminationStack.back() &&
3240 isa<Instruction>(Def) && !FromStore)
3241 markInstructionForDeletion(cast<Instruction>(Def));
3244 // At this point, we know it is a Use we are trying to possibly
3247 assert(isa<Instruction>(U->get()) &&
3248 "Current def should have been an instruction");
3249 assert(isa<Instruction>(U->getUser()) &&
3250 "Current user should have been an instruction");
3252 // If the thing we are replacing into is already marked to be dead,
3253 // this use is dead. Note that this is true regardless of whether
3254 // we have anything dominating the use or not. We do this here
3255 // because we are already walking all the uses anyway.
3256 Instruction *InstUse = cast<Instruction>(U->getUser());
3257 if (InstructionsToErase.count(InstUse)) {
3258 auto &UseCount = UseCounts[U->get()];
3259 if (--UseCount == 0) {
3260 ProbablyDead.insert(cast<Instruction>(U->get()));
3264 // If we get to this point, and the stack is empty we must have a use
3265 // with nothing we can use to eliminate this use, so just skip it.
3266 if (EliminationStack.empty())
3269 Value *DominatingLeader = EliminationStack.back();
3271 // Don't replace our existing users with ourselves.
3272 if (U->get() == DominatingLeader)
3274 DEBUG(dbgs() << "Found replacement " << *DominatingLeader << " for "
3275 << *U->get() << " in " << *(U->getUser()) << "\n");
3277 // If we replaced something in an instruction, handle the patching of
3278 // metadata. Skip this if we are replacing predicateinfo with its
3279 // original operand, as we already know we can just drop it.
3280 auto *ReplacedInst = cast<Instruction>(U->get());
3281 auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst);
3282 if (!PI || DominatingLeader != PI->OriginalOp)
3283 patchReplacementInstruction(ReplacedInst, DominatingLeader);
3284 U->set(DominatingLeader);
3285 // This is now a use of the dominating leader, which means if the
3286 // dominating leader was dead, it's now live!
3287 auto &LeaderUseCount = UseCounts[DominatingLeader];
3288 // It's about to be alive again.
3289 if (LeaderUseCount == 0 && isa<Instruction>(DominatingLeader))
3290 ProbablyDead.erase(cast<Instruction>(DominatingLeader));
3292 AnythingReplaced = true;
3297 // At this point, anything still in the ProbablyDead set is actually dead if
3298 // would be trivially dead.
3299 for (auto *I : ProbablyDead)
3300 if (wouldInstructionBeTriviallyDead(I))
3301 markInstructionForDeletion(I);
3303 // Cleanup the congruence class.
3304 CongruenceClass::MemberSet MembersLeft;
3305 for (auto *Member : *CC)
3306 if (!isa<Instruction>(Member) ||
3307 !InstructionsToErase.count(cast<Instruction>(Member)))
3308 MembersLeft.insert(Member);
3309 CC->swap(MembersLeft);
3311 // If we have possible dead stores to look at, try to eliminate them.
3312 if (CC->getStoreCount() > 0) {
3313 convertClassToLoadsAndStores(*CC, PossibleDeadStores);
3314 std::sort(PossibleDeadStores.begin(), PossibleDeadStores.end());
3315 ValueDFSStack EliminationStack;
3316 for (auto &VD : PossibleDeadStores) {
3317 int MemberDFSIn = VD.DFSIn;
3318 int MemberDFSOut = VD.DFSOut;
3319 Instruction *Member = cast<Instruction>(VD.Def.getPointer());
3320 if (EliminationStack.empty() ||
3321 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)) {
3322 // Sync to our current scope.
3323 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
3324 if (EliminationStack.empty()) {
3325 EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);
3329 // We already did load elimination, so nothing to do here.
3330 if (isa<LoadInst>(Member))
3332 assert(!EliminationStack.empty());
3333 Instruction *Leader = cast<Instruction>(EliminationStack.back());
3335 assert(DT->dominates(Leader->getParent(), Member->getParent()));
3336 // Member is dominater by Leader, and thus dead
3337 DEBUG(dbgs() << "Marking dead store " << *Member
3338 << " that is dominated by " << *Leader << "\n");
3339 markInstructionForDeletion(Member);
3346 return AnythingReplaced;
3349 // This function provides global ranking of operations so that we can place them
3350 // in a canonical order. Note that rank alone is not necessarily enough for a
3351 // complete ordering, as constants all have the same rank. However, generally,
3352 // we will simplify an operation with all constants so that it doesn't matter
3353 // what order they appear in.
3354 unsigned int NewGVN::getRank(const Value *V) const {
3355 // Prefer undef to anything else
3356 if (isa<UndefValue>(V))
3358 if (isa<Constant>(V))
3360 else if (auto *A = dyn_cast<Argument>(V))
3361 return 2 + A->getArgNo();
3363 // Need to shift the instruction DFS by number of arguments + 3 to account for
3364 // the constant and argument ranking above.
3365 unsigned Result = InstrToDFSNum(V);
3367 return 3 + NumFuncArgs + Result;
3368 // Unreachable or something else, just return a really large number.
3372 // This is a function that says whether two commutative operations should
3373 // have their order swapped when canonicalizing.
3374 bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
3375 // Because we only care about a total ordering, and don't rewrite expressions
3376 // in this order, we order by rank, which will give a strict weak ordering to
3377 // everything but constants, and then we order by pointer address.
3378 return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);
3381 class NewGVNLegacyPass : public FunctionPass {
3383 static char ID; // Pass identification, replacement for typeid.
3384 NewGVNLegacyPass() : FunctionPass(ID) {
3385 initializeNewGVNLegacyPassPass(*PassRegistry::getPassRegistry());
3387 bool runOnFunction(Function &F) override;
3390 void getAnalysisUsage(AnalysisUsage &AU) const override {
3391 AU.addRequired<AssumptionCacheTracker>();
3392 AU.addRequired<DominatorTreeWrapperPass>();
3393 AU.addRequired<TargetLibraryInfoWrapperPass>();
3394 AU.addRequired<MemorySSAWrapperPass>();
3395 AU.addRequired<AAResultsWrapperPass>();
3396 AU.addPreserved<DominatorTreeWrapperPass>();
3397 AU.addPreserved<GlobalsAAWrapperPass>();
3401 bool NewGVNLegacyPass::runOnFunction(Function &F) {
3402 if (skipFunction(F))
3404 return NewGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
3405 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
3406 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
3407 &getAnalysis<AAResultsWrapperPass>().getAAResults(),
3408 &getAnalysis<MemorySSAWrapperPass>().getMSSA(),
3409 F.getParent()->getDataLayout())
3413 INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering",
3415 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
3416 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
3417 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
3418 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
3419 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
3420 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
3421 INITIALIZE_PASS_END(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false,
3424 char NewGVNLegacyPass::ID = 0;
3426 // createGVNPass - The public interface to this file.
3427 FunctionPass *llvm::createNewGVNPass() { return new NewGVNLegacyPass(); }
3429 PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
3430 // Apparently the order in which we get these results matter for
3431 // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
3432 // the same order here, just in case.
3433 auto &AC = AM.getResult<AssumptionAnalysis>(F);
3434 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
3435 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
3436 auto &AA = AM.getResult<AAManager>(F);
3437 auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
3439 NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getParent()->getDataLayout())
3442 return PreservedAnalyses::all();
3443 PreservedAnalyses PA;
3444 PA.preserve<DominatorTreeAnalysis>();
3445 PA.preserve<GlobalsAA>();