1 //===---- NewGVN.cpp - Global Value Numbering Pass --------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 /// This file implements the new LLVM's Global Value Numbering pass.
11 /// GVN partitions values computed by a function into congruence classes.
12 /// Values ending up in the same congruence class are guaranteed to be the same
13 /// for every execution of the program. In that respect, congruency is a
14 /// compile-time approximation of equivalence of values at runtime.
15 /// The algorithm implemented here uses a sparse formulation and it's based
16 /// on the ideas described in the paper:
17 /// "A Sparse Algorithm for Predicated Global Value Numbering" from
20 //===----------------------------------------------------------------------===//
22 #include "llvm/Transforms/Scalar/NewGVN.h"
23 #include "llvm/ADT/BitVector.h"
24 #include "llvm/ADT/DenseMap.h"
25 #include "llvm/ADT/DenseSet.h"
26 #include "llvm/ADT/DepthFirstIterator.h"
27 #include "llvm/ADT/Hashing.h"
28 #include "llvm/ADT/MapVector.h"
29 #include "llvm/ADT/PostOrderIterator.h"
30 #include "llvm/ADT/STLExtras.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallSet.h"
33 #include "llvm/ADT/SparseBitVector.h"
34 #include "llvm/ADT/Statistic.h"
35 #include "llvm/ADT/TinyPtrVector.h"
36 #include "llvm/Analysis/AliasAnalysis.h"
37 #include "llvm/Analysis/AssumptionCache.h"
38 #include "llvm/Analysis/CFG.h"
39 #include "llvm/Analysis/CFGPrinter.h"
40 #include "llvm/Analysis/ConstantFolding.h"
41 #include "llvm/Analysis/GlobalsModRef.h"
42 #include "llvm/Analysis/InstructionSimplify.h"
43 #include "llvm/Analysis/Loads.h"
44 #include "llvm/Analysis/MemoryBuiltins.h"
45 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
46 #include "llvm/Analysis/MemoryLocation.h"
47 #include "llvm/Analysis/PHITransAddr.h"
48 #include "llvm/Analysis/TargetLibraryInfo.h"
49 #include "llvm/Analysis/ValueTracking.h"
50 #include "llvm/IR/DataLayout.h"
51 #include "llvm/IR/Dominators.h"
52 #include "llvm/IR/GlobalVariable.h"
53 #include "llvm/IR/IRBuilder.h"
54 #include "llvm/IR/IntrinsicInst.h"
55 #include "llvm/IR/LLVMContext.h"
56 #include "llvm/IR/Metadata.h"
57 #include "llvm/IR/PatternMatch.h"
58 #include "llvm/IR/PredIteratorCache.h"
59 #include "llvm/IR/Type.h"
60 #include "llvm/Support/Allocator.h"
61 #include "llvm/Support/CommandLine.h"
62 #include "llvm/Support/Debug.h"
63 #include "llvm/Transforms/Scalar.h"
64 #include "llvm/Transforms/Scalar/GVNExpression.h"
65 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
66 #include "llvm/Transforms/Utils/Local.h"
67 #include "llvm/Transforms/Utils/MemorySSA.h"
68 #include "llvm/Transforms/Utils/SSAUpdater.h"
69 #include <unordered_map>
73 using namespace PatternMatch;
74 using namespace llvm::GVNExpression;
76 #define DEBUG_TYPE "newgvn"
78 STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
79 STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
80 STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
81 STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
82 STATISTIC(NumGVNMaxIterations,
83 "Maximum Number of iterations it took to converge GVN");
84 STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
85 STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
86 STATISTIC(NumGVNAvoidedSortedLeaderChanges,
87 "Number of avoided sorted leader changes");
88 STATISTIC(NumGVNNotMostDominatingLeader,
89 "Number of times a member dominated it's new classes' leader");
91 //===----------------------------------------------------------------------===//
93 //===----------------------------------------------------------------------===//
97 namespace GVNExpression {
98 Expression::~Expression() = default;
99 BasicExpression::~BasicExpression() = default;
100 CallExpression::~CallExpression() = default;
101 LoadExpression::~LoadExpression() = default;
102 StoreExpression::~StoreExpression() = default;
103 AggregateValueExpression::~AggregateValueExpression() = default;
104 PHIExpression::~PHIExpression() = default;
108 // Congruence classes represent the set of expressions/instructions
109 // that are all the same *during some scope in the function*.
110 // That is, because of the way we perform equality propagation, and
111 // because of memory value numbering, it is not correct to assume
112 // you can willy-nilly replace any member with any other at any
113 // point in the function.
115 // For any Value in the Member set, it is valid to replace any dominated member
118 // Every congruence class has a leader, and the leader is used to
119 // symbolize instructions in a canonical way (IE every operand of an
120 // instruction that is a member of the same congruence class will
121 // always be replaced with leader during symbolization).
122 // To simplify symbolization, we keep the leader as a constant if class can be
123 // proved to be a constant value.
124 // Otherwise, the leader is a randomly chosen member of the value set, it does
125 // not matter which one is chosen.
126 // Each congruence class also has a defining expression,
127 // though the expression may be null. If it exists, it can be used for forward
128 // propagation and reassociation of values.
130 struct CongruenceClass {
131 using MemberSet = SmallPtrSet<Value *, 4>;
133 // Representative leader.
134 Value *RepLeader = nullptr;
135 // Defining Expression.
136 const Expression *DefiningExpr = nullptr;
137 // Actual members of this class.
140 // True if this class has no members left. This is mainly used for assertion
141 // purposes, and for skipping empty classes.
144 // Number of stores in this congruence class.
145 // This is used so we can detect store equivalence changes properly.
148 // The most dominating leader after our current leader, because the member set
149 // is not sorted and is expensive to keep sorted all the time.
150 std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
152 explicit CongruenceClass(unsigned ID) : ID(ID) {}
153 CongruenceClass(unsigned ID, Value *Leader, const Expression *E)
154 : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
158 template <> struct DenseMapInfo<const Expression *> {
159 static const Expression *getEmptyKey() {
160 auto Val = static_cast<uintptr_t>(-1);
161 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
162 return reinterpret_cast<const Expression *>(Val);
164 static const Expression *getTombstoneKey() {
165 auto Val = static_cast<uintptr_t>(~1U);
166 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
167 return reinterpret_cast<const Expression *>(Val);
169 static unsigned getHashValue(const Expression *V) {
170 return static_cast<unsigned>(V->getHashValue());
172 static bool isEqual(const Expression *LHS, const Expression *RHS) {
175 if (LHS == getTombstoneKey() || RHS == getTombstoneKey() ||
176 LHS == getEmptyKey() || RHS == getEmptyKey())
181 } // end namespace llvm
183 class NewGVN : public FunctionPass {
185 const DataLayout *DL;
186 const TargetLibraryInfo *TLI;
190 MemorySSAWalker *MSSAWalker;
191 BumpPtrAllocator ExpressionAllocator;
192 ArrayRecycler<Value *> ArgRecycler;
194 // Congruence class info.
195 CongruenceClass *InitialClass;
196 std::vector<CongruenceClass *> CongruenceClasses;
197 unsigned NextCongruenceNum;
200 DenseMap<Value *, CongruenceClass *> ValueToClass;
201 DenseMap<Value *, const Expression *> ValueToExpression;
203 // A table storing which memorydefs/phis represent a memory state provably
204 // equivalent to another memory state.
205 // We could use the congruence class machinery, but the MemoryAccess's are
206 // abstract memory states, so they can only ever be equivalent to each other,
207 // and not to constants, etc.
208 DenseMap<const MemoryAccess *, MemoryAccess *> MemoryAccessEquiv;
210 // Expression to class mapping.
211 using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
212 ExpressionClassMap ExpressionToClass;
214 // Which values have changed as a result of leader changes.
215 SmallPtrSet<Value *, 8> LeaderChanges;
217 // Reachability info.
218 using BlockEdge = BasicBlockEdge;
219 DenseSet<BlockEdge> ReachableEdges;
220 SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
222 // This is a bitvector because, on larger functions, we may have
223 // thousands of touched instructions at once (entire blocks,
224 // instructions with hundreds of uses, etc). Even with optimization
225 // for when we mark whole blocks as touched, when this was a
226 // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
227 // the time in GVN just managing this list. The bitvector, on the
228 // other hand, efficiently supports test/set/clear of both
229 // individual and ranges, as well as "find next element" This
230 // enables us to use it as a worklist with essentially 0 cost.
231 BitVector TouchedInstructions;
233 DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
234 DenseMap<const DomTreeNode *, std::pair<unsigned, unsigned>>
238 // Debugging for how many times each block and instruction got processed.
239 DenseMap<const Value *, unsigned> ProcessedCount;
243 DenseMap<const BasicBlock *, std::pair<int, int>> DFSDomMap;
244 DenseMap<const Value *, unsigned> InstrDFS;
245 SmallVector<Value *, 32> DFSToInstr;
248 SmallPtrSet<Instruction *, 8> InstructionsToErase;
251 static char ID; // Pass identification, replacement for typeid.
252 NewGVN() : FunctionPass(ID) {
253 initializeNewGVNPass(*PassRegistry::getPassRegistry());
256 bool runOnFunction(Function &F) override;
257 bool runGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
258 TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA);
261 // This transformation requires dominator postdominator info.
262 void getAnalysisUsage(AnalysisUsage &AU) const override {
263 AU.addRequired<AssumptionCacheTracker>();
264 AU.addRequired<DominatorTreeWrapperPass>();
265 AU.addRequired<TargetLibraryInfoWrapperPass>();
266 AU.addRequired<MemorySSAWrapperPass>();
267 AU.addRequired<AAResultsWrapperPass>();
269 AU.addPreserved<DominatorTreeWrapperPass>();
270 AU.addPreserved<GlobalsAAWrapperPass>();
273 // Expression handling.
274 const Expression *createExpression(Instruction *, const BasicBlock *);
275 const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,
277 PHIExpression *createPHIExpression(Instruction *);
278 const VariableExpression *createVariableExpression(Value *);
279 const ConstantExpression *createConstantExpression(Constant *);
280 const Expression *createVariableOrConstant(Value *V, const BasicBlock *B);
281 const UnknownExpression *createUnknownExpression(Instruction *);
282 const StoreExpression *createStoreExpression(StoreInst *, MemoryAccess *,
284 LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
285 MemoryAccess *, const BasicBlock *);
287 const CallExpression *createCallExpression(CallInst *, MemoryAccess *,
289 const AggregateValueExpression *
290 createAggregateValueExpression(Instruction *, const BasicBlock *);
291 bool setBasicExpressionInfo(Instruction *, BasicExpression *,
294 // Congruence class handling.
295 CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
296 auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E);
297 CongruenceClasses.emplace_back(result);
301 CongruenceClass *createSingletonCongruenceClass(Value *Member) {
302 CongruenceClass *CClass = createCongruenceClass(Member, nullptr);
303 CClass->Members.insert(Member);
304 ValueToClass[Member] = CClass;
307 void initializeCongruenceClasses(Function &F);
309 // Value number an Instruction or MemoryPhi.
310 void valueNumberMemoryPhi(MemoryPhi *);
311 void valueNumberInstruction(Instruction *);
313 // Symbolic evaluation.
314 const Expression *checkSimplificationResults(Expression *, Instruction *,
316 const Expression *performSymbolicEvaluation(Value *, const BasicBlock *);
317 const Expression *performSymbolicLoadEvaluation(Instruction *,
319 const Expression *performSymbolicStoreEvaluation(Instruction *,
321 const Expression *performSymbolicCallEvaluation(Instruction *,
323 const Expression *performSymbolicPHIEvaluation(Instruction *,
325 bool setMemoryAccessEquivTo(MemoryAccess *From, MemoryAccess *To);
326 const Expression *performSymbolicAggrValueEvaluation(Instruction *,
329 // Congruence finding.
330 // Templated to allow them to work both on BB's and BB-edges.
332 Value *lookupOperandLeader(Value *, const User *, const T &) const;
333 void performCongruenceFinding(Instruction *, const Expression *);
334 void moveValueToNewCongruenceClass(Instruction *, CongruenceClass *,
336 // Reachability handling.
337 void updateReachableEdge(BasicBlock *, BasicBlock *);
338 void processOutgoingEdges(TerminatorInst *, BasicBlock *);
339 bool isOnlyReachableViaThisEdge(const BasicBlockEdge &) const;
340 Value *findConditionEquivalence(Value *, BasicBlock *) const;
341 MemoryAccess *lookupMemoryAccessEquiv(MemoryAccess *) const;
345 void convertDenseToDFSOrdered(CongruenceClass::MemberSet &,
346 SmallVectorImpl<ValueDFS> &);
348 bool eliminateInstructions(Function &);
349 void replaceInstruction(Instruction *, Value *);
350 void markInstructionForDeletion(Instruction *);
351 void deleteInstructionsInBlock(BasicBlock *);
353 // New instruction creation.
354 void handleNewInstruction(Instruction *){};
356 // Various instruction touch utilities
357 void markUsersTouched(Value *);
358 void markMemoryUsersTouched(MemoryAccess *);
359 void markLeaderChangeTouched(CongruenceClass *CC);
362 void cleanupTables();
363 std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
364 void updateProcessedCount(Value *V);
365 void verifyMemoryCongruency() const;
366 bool singleReachablePHIPath(const MemoryAccess *, const MemoryAccess *) const;
371 // createGVNPass - The public interface to this file.
372 FunctionPass *llvm::createNewGVNPass() { return new NewGVN(); }
374 template <typename T>
375 static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
376 if ((!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS)) ||
377 !LHS.BasicExpression::equals(RHS)) {
379 } else if (const auto *L = dyn_cast<LoadExpression>(&RHS)) {
380 if (LHS.getDefiningAccess() != L->getDefiningAccess())
382 } else if (const auto *S = dyn_cast<StoreExpression>(&RHS)) {
383 if (LHS.getDefiningAccess() != S->getDefiningAccess())
389 bool LoadExpression::equals(const Expression &Other) const {
390 return equalsLoadStoreHelper(*this, Other);
393 bool StoreExpression::equals(const Expression &Other) const {
394 return equalsLoadStoreHelper(*this, Other);
398 static std::string getBlockName(const BasicBlock *B) {
399 return DOTGraphTraits<const Function *>::getSimpleNodeLabel(B, nullptr);
403 INITIALIZE_PASS_BEGIN(NewGVN, "newgvn", "Global Value Numbering", false, false)
404 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
405 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
406 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
407 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
408 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
409 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
410 INITIALIZE_PASS_END(NewGVN, "newgvn", "Global Value Numbering", false, false)
412 PHIExpression *NewGVN::createPHIExpression(Instruction *I) {
413 BasicBlock *PHIBlock = I->getParent();
414 auto *PN = cast<PHINode>(I);
416 new (ExpressionAllocator) PHIExpression(PN->getNumOperands(), PHIBlock);
418 E->allocateOperands(ArgRecycler, ExpressionAllocator);
419 E->setType(I->getType());
420 E->setOpcode(I->getOpcode());
422 auto ReachablePhiArg = [&](const Use &U) {
423 return ReachableBlocks.count(PN->getIncomingBlock(U));
426 // Filter out unreachable operands
427 auto Filtered = make_filter_range(PN->operands(), ReachablePhiArg);
429 std::transform(Filtered.begin(), Filtered.end(), op_inserter(E),
430 [&](const Use &U) -> Value * {
431 // Don't try to transform self-defined phis.
434 const BasicBlockEdge BBE(PN->getIncomingBlock(U), PHIBlock);
435 return lookupOperandLeader(U, I, BBE);
440 // Set basic expression info (Arguments, type, opcode) for Expression
441 // E from Instruction I in block B.
442 bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E,
443 const BasicBlock *B) {
444 bool AllConstant = true;
445 if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
446 E->setType(GEP->getSourceElementType());
448 E->setType(I->getType());
449 E->setOpcode(I->getOpcode());
450 E->allocateOperands(ArgRecycler, ExpressionAllocator);
452 // Transform the operand array into an operand leader array, and keep track of
453 // whether all members are constant.
454 std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {
455 auto Operand = lookupOperandLeader(O, I, B);
456 AllConstant &= isa<Constant>(Operand);
463 const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
464 Value *Arg1, Value *Arg2,
465 const BasicBlock *B) {
466 auto *E = new (ExpressionAllocator) BasicExpression(2);
469 E->setOpcode(Opcode);
470 E->allocateOperands(ArgRecycler, ExpressionAllocator);
471 if (Instruction::isCommutative(Opcode)) {
472 // Ensure that commutative instructions that only differ by a permutation
473 // of their operands get the same value number by sorting the operand value
474 // numbers. Since all commutative instructions have two operands it is more
475 // efficient to sort by hand rather than using, say, std::sort.
477 std::swap(Arg1, Arg2);
479 E->op_push_back(lookupOperandLeader(Arg1, nullptr, B));
480 E->op_push_back(lookupOperandLeader(Arg2, nullptr, B));
482 Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), *DL, TLI,
484 if (const Expression *SimplifiedE = checkSimplificationResults(E, nullptr, V))
489 // Take a Value returned by simplification of Expression E/Instruction
490 // I, and see if it resulted in a simpler expression. If so, return
492 // TODO: Once finished, this should not take an Instruction, we only
493 // use it for printing.
494 const Expression *NewGVN::checkSimplificationResults(Expression *E,
495 Instruction *I, Value *V) {
498 if (auto *C = dyn_cast<Constant>(V)) {
500 DEBUG(dbgs() << "Simplified " << *I << " to "
501 << " constant " << *C << "\n");
502 NumGVNOpsSimplified++;
503 assert(isa<BasicExpression>(E) &&
504 "We should always have had a basic expression here");
506 cast<BasicExpression>(E)->deallocateOperands(ArgRecycler);
507 ExpressionAllocator.Deallocate(E);
508 return createConstantExpression(C);
509 } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
511 DEBUG(dbgs() << "Simplified " << *I << " to "
512 << " variable " << *V << "\n");
513 cast<BasicExpression>(E)->deallocateOperands(ArgRecycler);
514 ExpressionAllocator.Deallocate(E);
515 return createVariableExpression(V);
518 CongruenceClass *CC = ValueToClass.lookup(V);
519 if (CC && CC->DefiningExpr) {
521 DEBUG(dbgs() << "Simplified " << *I << " to "
522 << " expression " << *V << "\n");
523 NumGVNOpsSimplified++;
524 assert(isa<BasicExpression>(E) &&
525 "We should always have had a basic expression here");
526 cast<BasicExpression>(E)->deallocateOperands(ArgRecycler);
527 ExpressionAllocator.Deallocate(E);
528 return CC->DefiningExpr;
533 const Expression *NewGVN::createExpression(Instruction *I,
534 const BasicBlock *B) {
536 auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
538 bool AllConstant = setBasicExpressionInfo(I, E, B);
540 if (I->isCommutative()) {
541 // Ensure that commutative instructions that only differ by a permutation
542 // of their operands get the same value number by sorting the operand value
543 // numbers. Since all commutative instructions have two operands it is more
544 // efficient to sort by hand rather than using, say, std::sort.
545 assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
546 if (E->getOperand(0) > E->getOperand(1))
547 E->swapOperands(0, 1);
550 // Perform simplificaiton
551 // TODO: Right now we only check to see if we get a constant result.
552 // We may get a less than constant, but still better, result for
557 // We should handle this by simply rewriting the expression.
558 if (auto *CI = dyn_cast<CmpInst>(I)) {
559 // Sort the operand value numbers so x<y and y>x get the same value
561 CmpInst::Predicate Predicate = CI->getPredicate();
562 if (E->getOperand(0) > E->getOperand(1)) {
563 E->swapOperands(0, 1);
564 Predicate = CmpInst::getSwappedPredicate(Predicate);
566 E->setOpcode((CI->getOpcode() << 8) | Predicate);
567 // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands
568 // TODO: Since we noop bitcasts, we may need to check types before
569 // simplifying, so that we don't end up simplifying based on a wrong
570 // type assumption. We should clean this up so we can use constants of the
573 assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
574 "Wrong types on cmp instruction");
575 if ((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
576 E->getOperand(1)->getType() == I->getOperand(1)->getType())) {
577 Value *V = SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1),
579 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
582 } else if (isa<SelectInst>(I)) {
583 if (isa<Constant>(E->getOperand(0)) ||
584 (E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
585 E->getOperand(2)->getType() == I->getOperand(2)->getType())) {
586 Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
587 E->getOperand(2), *DL, TLI, DT, AC);
588 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
591 } else if (I->isBinaryOp()) {
592 Value *V = SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1),
594 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
596 } else if (auto *BI = dyn_cast<BitCastInst>(I)) {
597 Value *V = SimplifyInstruction(BI, *DL, TLI, DT, AC);
598 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
600 } else if (isa<GetElementPtrInst>(I)) {
601 Value *V = SimplifyGEPInst(E->getType(),
602 ArrayRef<Value *>(E->op_begin(), E->op_end()),
604 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
606 } else if (AllConstant) {
607 // We don't bother trying to simplify unless all of the operands
609 // TODO: There are a lot of Simplify*'s we could call here, if we
610 // wanted to. The original motivating case for this code was a
611 // zext i1 false to i8, which we don't have an interface to
612 // simplify (IE there is no SimplifyZExt).
614 SmallVector<Constant *, 8> C;
615 for (Value *Arg : E->operands())
616 C.emplace_back(cast<Constant>(Arg));
618 if (Value *V = ConstantFoldInstOperands(I, C, *DL, TLI))
619 if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
625 const AggregateValueExpression *
626 NewGVN::createAggregateValueExpression(Instruction *I, const BasicBlock *B) {
627 if (auto *II = dyn_cast<InsertValueInst>(I)) {
628 auto *E = new (ExpressionAllocator)
629 AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
630 setBasicExpressionInfo(I, E, B);
631 E->allocateIntOperands(ExpressionAllocator);
632 std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E));
634 } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
635 auto *E = new (ExpressionAllocator)
636 AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
637 setBasicExpressionInfo(EI, E, B);
638 E->allocateIntOperands(ExpressionAllocator);
639 std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E));
642 llvm_unreachable("Unhandled type of aggregate value operation");
645 const VariableExpression *NewGVN::createVariableExpression(Value *V) {
646 auto *E = new (ExpressionAllocator) VariableExpression(V);
647 E->setOpcode(V->getValueID());
651 const Expression *NewGVN::createVariableOrConstant(Value *V,
652 const BasicBlock *B) {
653 auto Leader = lookupOperandLeader(V, nullptr, B);
654 if (auto *C = dyn_cast<Constant>(Leader))
655 return createConstantExpression(C);
656 return createVariableExpression(Leader);
659 const ConstantExpression *NewGVN::createConstantExpression(Constant *C) {
660 auto *E = new (ExpressionAllocator) ConstantExpression(C);
661 E->setOpcode(C->getValueID());
665 const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) {
666 auto *E = new (ExpressionAllocator) UnknownExpression(I);
667 E->setOpcode(I->getOpcode());
671 const CallExpression *NewGVN::createCallExpression(CallInst *CI,
673 const BasicBlock *B) {
674 // FIXME: Add operand bundles for calls.
676 new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, HV);
677 setBasicExpressionInfo(CI, E, B);
681 // See if we have a congruence class and leader for this operand, and if so,
682 // return it. Otherwise, return the operand itself.
684 Value *NewGVN::lookupOperandLeader(Value *V, const User *U, const T &B) const {
685 CongruenceClass *CC = ValueToClass.lookup(V);
686 if (CC && (CC != InitialClass))
687 return CC->RepLeader;
691 MemoryAccess *NewGVN::lookupMemoryAccessEquiv(MemoryAccess *MA) const {
692 MemoryAccess *Result = MemoryAccessEquiv.lookup(MA);
693 return Result ? Result : MA;
696 LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
697 LoadInst *LI, MemoryAccess *DA,
698 const BasicBlock *B) {
699 auto *E = new (ExpressionAllocator) LoadExpression(1, LI, DA);
700 E->allocateOperands(ArgRecycler, ExpressionAllocator);
701 E->setType(LoadType);
703 // Give store and loads same opcode so they value number together.
705 E->op_push_back(lookupOperandLeader(PointerOp, LI, B));
707 E->setAlignment(LI->getAlignment());
709 // TODO: Value number heap versions. We may be able to discover
710 // things alias analysis can't on it's own (IE that a store and a
711 // load have the same value, and thus, it isn't clobbering the load).
715 const StoreExpression *NewGVN::createStoreExpression(StoreInst *SI,
717 const BasicBlock *B) {
719 new (ExpressionAllocator) StoreExpression(SI->getNumOperands(), SI, DA);
720 E->allocateOperands(ArgRecycler, ExpressionAllocator);
721 E->setType(SI->getValueOperand()->getType());
723 // Give store and loads same opcode so they value number together.
725 E->op_push_back(lookupOperandLeader(SI->getPointerOperand(), SI, B));
727 // TODO: Value number heap versions. We may be able to discover
728 // things alias analysis can't on it's own (IE that a store and a
729 // load have the same value, and thus, it isn't clobbering the load).
733 // Utility function to check whether the congruence class has a member other
734 // than the given instruction.
735 bool hasMemberOtherThanUs(const CongruenceClass *CC, Instruction *I) {
736 // Either it has more than one store, in which case it must contain something
737 // other than us (because it's indexed by value), or if it only has one store
738 // right now, that member should not be us.
739 return CC->StoreCount > 1 || CC->Members.count(I) == 0;
742 const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I,
743 const BasicBlock *B) {
744 // Unlike loads, we never try to eliminate stores, so we do not check if they
745 // are simple and avoid value numbering them.
746 auto *SI = cast<StoreInst>(I);
747 MemoryAccess *StoreAccess = MSSA->getMemoryAccess(SI);
748 // See if we are defined by a previous store expression, it already has a
749 // value, and it's the same value as our current store. FIXME: Right now, we
750 // only do this for simple stores, we should expand to cover memcpys, etc.
751 if (SI->isSimple()) {
752 // Get the expression, if any, for the RHS of the MemoryDef.
753 MemoryAccess *StoreRHS = lookupMemoryAccessEquiv(
754 cast<MemoryDef>(StoreAccess)->getDefiningAccess());
755 const Expression *OldStore = createStoreExpression(SI, StoreRHS, B);
756 CongruenceClass *CC = ExpressionToClass.lookup(OldStore);
757 // Basically, check if the congruence class the store is in is defined by a
758 // store that isn't us, and has the same value. MemorySSA takes care of
759 // ensuring the store has the same memory state as us already.
760 if (CC && CC->DefiningExpr && isa<StoreExpression>(CC->DefiningExpr) &&
761 CC->RepLeader == lookupOperandLeader(SI->getValueOperand(), SI, B) &&
762 hasMemberOtherThanUs(CC, I))
763 return createStoreExpression(SI, StoreRHS, B);
766 return createStoreExpression(SI, StoreAccess, B);
769 const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I,
770 const BasicBlock *B) {
771 auto *LI = cast<LoadInst>(I);
773 // We can eliminate in favor of non-simple loads, but we won't be able to
774 // eliminate the loads themselves.
778 Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand(), I, B);
779 // Load of undef is undef.
780 if (isa<UndefValue>(LoadAddressLeader))
781 return createConstantExpression(UndefValue::get(LI->getType()));
783 MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(I);
785 if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
786 if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
787 Instruction *DefiningInst = MD->getMemoryInst();
788 // If the defining instruction is not reachable, replace with undef.
789 if (!ReachableBlocks.count(DefiningInst->getParent()))
790 return createConstantExpression(UndefValue::get(LI->getType()));
794 const Expression *E =
795 createLoadExpression(LI->getType(), LI->getPointerOperand(), LI,
796 lookupMemoryAccessEquiv(DefiningAccess), B);
800 // Evaluate read only and pure calls, and create an expression result.
801 const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I,
802 const BasicBlock *B) {
803 auto *CI = cast<CallInst>(I);
804 if (AA->doesNotAccessMemory(CI))
805 return createCallExpression(CI, nullptr, B);
806 if (AA->onlyReadsMemory(CI)) {
807 MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI);
808 return createCallExpression(CI, lookupMemoryAccessEquiv(DefiningAccess), B);
813 // Update the memory access equivalence table to say that From is equal to To,
814 // and return true if this is different from what already existed in the table.
815 bool NewGVN::setMemoryAccessEquivTo(MemoryAccess *From, MemoryAccess *To) {
816 DEBUG(dbgs() << "Setting " << *From << " equivalent to ");
818 DEBUG(dbgs() << "itself");
820 DEBUG(dbgs() << *To);
821 DEBUG(dbgs() << "\n");
822 auto LookupResult = MemoryAccessEquiv.find(From);
823 bool Changed = false;
824 // If it's already in the table, see if the value changed.
825 if (LookupResult != MemoryAccessEquiv.end()) {
826 if (To && LookupResult->second != To) {
827 // It wasn't equivalent before, and now it is.
828 LookupResult->second = To;
831 // It used to be equivalent to something, and now it's not.
832 MemoryAccessEquiv.erase(LookupResult);
837 "Memory equivalence should never change from nothing to something");
842 // Evaluate PHI nodes symbolically, and create an expression result.
843 const Expression *NewGVN::performSymbolicPHIEvaluation(Instruction *I,
844 const BasicBlock *B) {
845 auto *E = cast<PHIExpression>(createPHIExpression(I));
846 // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
848 // See if all arguaments are the same.
849 // We track if any were undef because they need special handling.
850 bool HasUndef = false;
851 auto Filtered = make_filter_range(E->operands(), [&](const Value *Arg) {
854 if (isa<UndefValue>(Arg)) {
860 // If we are left with no operands, it's undef
861 if (Filtered.begin() == Filtered.end()) {
862 DEBUG(dbgs() << "Simplified PHI node " << *I << " to undef"
864 E->deallocateOperands(ArgRecycler);
865 ExpressionAllocator.Deallocate(E);
866 return createConstantExpression(UndefValue::get(I->getType()));
868 Value *AllSameValue = *(Filtered.begin());
870 // Can't use std::equal here, sadly, because filter.begin moves.
871 if (llvm::all_of(Filtered, [AllSameValue](const Value *V) {
872 return V == AllSameValue;
874 // In LLVM's non-standard representation of phi nodes, it's possible to have
875 // phi nodes with cycles (IE dependent on other phis that are .... dependent
876 // on the original phi node), especially in weird CFG's where some arguments
877 // are unreachable, or uninitialized along certain paths. This can cause
878 // infinite loops during evaluation. We work around this by not trying to
879 // really evaluate them independently, but instead using a variable
880 // expression to say if one is equivalent to the other.
881 // We also special case undef, so that if we have an undef, we can't use the
882 // common value unless it dominates the phi block.
884 // Only have to check for instructions
885 if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))
886 if (!DT->dominates(AllSameInst, I))
891 DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
893 E->deallocateOperands(ArgRecycler);
894 ExpressionAllocator.Deallocate(E);
895 if (auto *C = dyn_cast<Constant>(AllSameValue))
896 return createConstantExpression(C);
897 return createVariableExpression(AllSameValue);
903 NewGVN::performSymbolicAggrValueEvaluation(Instruction *I,
904 const BasicBlock *B) {
905 if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
906 auto *II = dyn_cast<IntrinsicInst>(EI->getAggregateOperand());
907 if (II && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) {
909 // EI might be an extract from one of our recognised intrinsics. If it
910 // is we'll synthesize a semantically equivalent expression instead on
911 // an extract value expression.
912 switch (II->getIntrinsicID()) {
913 case Intrinsic::sadd_with_overflow:
914 case Intrinsic::uadd_with_overflow:
915 Opcode = Instruction::Add;
917 case Intrinsic::ssub_with_overflow:
918 case Intrinsic::usub_with_overflow:
919 Opcode = Instruction::Sub;
921 case Intrinsic::smul_with_overflow:
922 case Intrinsic::umul_with_overflow:
923 Opcode = Instruction::Mul;
930 // Intrinsic recognized. Grab its args to finish building the
932 assert(II->getNumArgOperands() == 2 &&
933 "Expect two args for recognised intrinsics.");
934 return createBinaryExpression(Opcode, EI->getType(),
935 II->getArgOperand(0),
936 II->getArgOperand(1), B);
941 return createAggregateValueExpression(I, B);
944 // Substitute and symbolize the value before value numbering.
945 const Expression *NewGVN::performSymbolicEvaluation(Value *V,
946 const BasicBlock *B) {
947 const Expression *E = nullptr;
948 if (auto *C = dyn_cast<Constant>(V))
949 E = createConstantExpression(C);
950 else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
951 E = createVariableExpression(V);
953 // TODO: memory intrinsics.
954 // TODO: Some day, we should do the forward propagation and reassociation
955 // parts of the algorithm.
956 auto *I = cast<Instruction>(V);
957 switch (I->getOpcode()) {
958 case Instruction::ExtractValue:
959 case Instruction::InsertValue:
960 E = performSymbolicAggrValueEvaluation(I, B);
962 case Instruction::PHI:
963 E = performSymbolicPHIEvaluation(I, B);
965 case Instruction::Call:
966 E = performSymbolicCallEvaluation(I, B);
968 case Instruction::Store:
969 E = performSymbolicStoreEvaluation(I, B);
971 case Instruction::Load:
972 E = performSymbolicLoadEvaluation(I, B);
974 case Instruction::BitCast: {
975 E = createExpression(I, B);
978 case Instruction::Add:
979 case Instruction::FAdd:
980 case Instruction::Sub:
981 case Instruction::FSub:
982 case Instruction::Mul:
983 case Instruction::FMul:
984 case Instruction::UDiv:
985 case Instruction::SDiv:
986 case Instruction::FDiv:
987 case Instruction::URem:
988 case Instruction::SRem:
989 case Instruction::FRem:
990 case Instruction::Shl:
991 case Instruction::LShr:
992 case Instruction::AShr:
993 case Instruction::And:
994 case Instruction::Or:
995 case Instruction::Xor:
996 case Instruction::ICmp:
997 case Instruction::FCmp:
998 case Instruction::Trunc:
999 case Instruction::ZExt:
1000 case Instruction::SExt:
1001 case Instruction::FPToUI:
1002 case Instruction::FPToSI:
1003 case Instruction::UIToFP:
1004 case Instruction::SIToFP:
1005 case Instruction::FPTrunc:
1006 case Instruction::FPExt:
1007 case Instruction::PtrToInt:
1008 case Instruction::IntToPtr:
1009 case Instruction::Select:
1010 case Instruction::ExtractElement:
1011 case Instruction::InsertElement:
1012 case Instruction::ShuffleVector:
1013 case Instruction::GetElementPtr:
1014 E = createExpression(I, B);
1023 // There is an edge from 'Src' to 'Dst'. Return true if every path from
1024 // the entry block to 'Dst' passes via this edge. In particular 'Dst'
1025 // must not be reachable via another edge from 'Src'.
1026 bool NewGVN::isOnlyReachableViaThisEdge(const BasicBlockEdge &E) const {
1028 // While in theory it is interesting to consider the case in which Dst has
1029 // more than one predecessor, because Dst might be part of a loop which is
1030 // only reachable from Src, in practice it is pointless since at the time
1031 // GVN runs all such loops have preheaders, which means that Dst will have
1032 // been changed to have only one predecessor, namely Src.
1033 const BasicBlock *Pred = E.getEnd()->getSinglePredecessor();
1034 const BasicBlock *Src = E.getStart();
1035 assert((!Pred || Pred == Src) && "No edge between these basic blocks!");
1037 return Pred != nullptr;
1040 void NewGVN::markUsersTouched(Value *V) {
1041 // Now mark the users as touched.
1042 for (auto *User : V->users()) {
1043 assert(isa<Instruction>(User) && "Use of value not within an instruction?");
1044 TouchedInstructions.set(InstrDFS[User]);
1048 void NewGVN::markMemoryUsersTouched(MemoryAccess *MA) {
1049 for (auto U : MA->users()) {
1050 if (auto *MUD = dyn_cast<MemoryUseOrDef>(U))
1051 TouchedInstructions.set(InstrDFS[MUD->getMemoryInst()]);
1053 TouchedInstructions.set(InstrDFS[U]);
1057 // Touch the instructions that need to be updated after a congruence class has a
1058 // leader change, and mark changed values.
1059 void NewGVN::markLeaderChangeTouched(CongruenceClass *CC) {
1060 for (auto M : CC->Members) {
1061 if (auto *I = dyn_cast<Instruction>(M))
1062 TouchedInstructions.set(InstrDFS[I]);
1063 LeaderChanges.insert(M);
1067 // Move a value, currently in OldClass, to be part of NewClass
1068 // Update OldClass for the move (including changing leaders, etc)
1069 void NewGVN::moveValueToNewCongruenceClass(Instruction *I,
1070 CongruenceClass *OldClass,
1071 CongruenceClass *NewClass) {
1072 DEBUG(dbgs() << "New congruence class for " << I << " is " << NewClass->ID
1075 if (I == OldClass->NextLeader.first)
1076 OldClass->NextLeader = {nullptr, ~0U};
1078 // It's possible, though unlikely, for us to discover equivalences such
1079 // that the current leader does not dominate the old one.
1080 // This statistic tracks how often this happens.
1081 // We assert on phi nodes when this happens, currently, for debugging, because
1082 // we want to make sure we name phi node cycles properly.
1083 if (isa<Instruction>(NewClass->RepLeader) && NewClass->RepLeader &&
1084 I != NewClass->RepLeader &&
1085 DT->properlyDominates(
1087 cast<Instruction>(NewClass->RepLeader)->getParent())) {
1088 ++NumGVNNotMostDominatingLeader;
1089 assert(!isa<PHINode>(I) &&
1090 "New class for instruction should not be dominated by instruction");
1093 if (NewClass->RepLeader != I) {
1094 auto DFSNum = InstrDFS.lookup(I);
1095 if (DFSNum < NewClass->NextLeader.second)
1096 NewClass->NextLeader = {I, DFSNum};
1099 OldClass->Members.erase(I);
1100 NewClass->Members.insert(I);
1101 if (isa<StoreInst>(I)) {
1102 --OldClass->StoreCount;
1103 assert(OldClass->StoreCount >= 0);
1104 ++NewClass->StoreCount;
1105 assert(NewClass->StoreCount > 0);
1108 ValueToClass[I] = NewClass;
1109 // See if we destroyed the class or need to swap leaders.
1110 if (OldClass->Members.empty() && OldClass != InitialClass) {
1111 if (OldClass->DefiningExpr) {
1112 OldClass->Dead = true;
1113 DEBUG(dbgs() << "Erasing expression " << OldClass->DefiningExpr
1114 << " from table\n");
1115 ExpressionToClass.erase(OldClass->DefiningExpr);
1117 } else if (OldClass->RepLeader == I) {
1118 // When the leader changes, the value numbering of
1119 // everything may change due to symbolization changes, so we need to
1121 DEBUG(dbgs() << "Leader change!\n");
1122 ++NumGVNLeaderChanges;
1123 // We don't need to sort members if there is only 1, and we don't care about
1124 // sorting the initial class because everything either gets out of it or is
1126 if (OldClass->Members.size() == 1 || OldClass == InitialClass) {
1127 OldClass->RepLeader = *(OldClass->Members.begin());
1128 } else if (OldClass->NextLeader.first) {
1129 ++NumGVNAvoidedSortedLeaderChanges;
1130 OldClass->RepLeader = OldClass->NextLeader.first;
1131 OldClass->NextLeader = {nullptr, ~0U};
1133 ++NumGVNSortedLeaderChanges;
1134 // TODO: If this ends up to slow, we can maintain a dual structure for
1135 // member testing/insertion, or keep things mostly sorted, and sort only
1137 std::pair<Value *, unsigned> MinDFS = {nullptr, ~0U};
1138 for (const auto X : OldClass->Members) {
1139 auto DFSNum = InstrDFS.lookup(X);
1140 if (DFSNum < MinDFS.second)
1141 MinDFS = {X, DFSNum};
1143 OldClass->RepLeader = MinDFS.first;
1145 markLeaderChangeTouched(OldClass);
1149 // Perform congruence finding on a given value numbering expression.
1150 void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
1151 ValueToExpression[I] = E;
1152 // This is guaranteed to return something, since it will at least find
1155 CongruenceClass *IClass = ValueToClass[I];
1156 assert(IClass && "Should have found a IClass");
1157 // Dead classes should have been eliminated from the mapping.
1158 assert(!IClass->Dead && "Found a dead class");
1160 CongruenceClass *EClass;
1161 if (const auto *VE = dyn_cast<VariableExpression>(E)) {
1162 EClass = ValueToClass[VE->getVariableValue()];
1164 auto lookupResult = ExpressionToClass.insert({E, nullptr});
1166 // If it's not in the value table, create a new congruence class.
1167 if (lookupResult.second) {
1168 CongruenceClass *NewClass = createCongruenceClass(nullptr, E);
1169 auto place = lookupResult.first;
1170 place->second = NewClass;
1172 // Constants and variables should always be made the leader.
1173 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
1174 NewClass->RepLeader = CE->getConstantValue();
1175 } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {
1176 StoreInst *SI = SE->getStoreInst();
1177 NewClass->RepLeader =
1178 lookupOperandLeader(SI->getValueOperand(), SI, SI->getParent());
1180 NewClass->RepLeader = I;
1182 assert(!isa<VariableExpression>(E) &&
1183 "VariableExpression should have been handled already");
1186 DEBUG(dbgs() << "Created new congruence class for " << *I
1187 << " using expression " << *E << " at " << NewClass->ID
1188 << " and leader " << *(NewClass->RepLeader) << "\n");
1189 DEBUG(dbgs() << "Hash value was " << E->getHashValue() << "\n");
1191 EClass = lookupResult.first->second;
1192 if (isa<ConstantExpression>(E))
1193 assert(isa<Constant>(EClass->RepLeader) &&
1194 "Any class with a constant expression should have a "
1197 assert(EClass && "Somehow don't have an eclass");
1199 assert(!EClass->Dead && "We accidentally looked up a dead class");
1202 bool ClassChanged = IClass != EClass;
1203 bool LeaderChanged = LeaderChanges.erase(I);
1204 if (ClassChanged || LeaderChanged) {
1205 DEBUG(dbgs() << "Found class " << EClass->ID << " for expression " << E
1209 moveValueToNewCongruenceClass(I, IClass, EClass);
1210 markUsersTouched(I);
1211 if (MemoryAccess *MA = MSSA->getMemoryAccess(I)) {
1212 // If this is a MemoryDef, we need to update the equivalence table. If
1213 // we determined the expression is congruent to a different memory
1214 // state, use that different memory state. If we determined it didn't,
1215 // we update that as well. Right now, we only support store
1217 if (!isa<MemoryUse>(MA) && isa<StoreExpression>(E) &&
1218 EClass->Members.size() != 1) {
1219 auto *DefAccess = cast<StoreExpression>(E)->getDefiningAccess();
1220 setMemoryAccessEquivTo(MA, DefAccess != MA ? DefAccess : nullptr);
1222 setMemoryAccessEquivTo(MA, nullptr);
1224 markMemoryUsersTouched(MA);
1226 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
1227 // There is, sadly, one complicating thing for stores. Stores do not
1228 // produce values, only consume them. However, in order to make loads and
1229 // stores value number the same, we ignore the value operand of the store.
1230 // But the value operand will still be the leader of our class, and thus, it
1231 // may change. Because the store is a use, the store will get reprocessed,
1232 // but nothing will change about it, and so nothing above will catch it
1233 // (since the class will not change). In order to make sure everything ends
1234 // up okay, we need to recheck the leader of the class. Since stores of
1235 // different values value number differently due to different memorydefs, we
1236 // are guaranteed the leader is always the same between stores in the same
1238 DEBUG(dbgs() << "Checking store leader\n");
1240 lookupOperandLeader(SI->getValueOperand(), SI, SI->getParent());
1241 if (EClass->RepLeader != ProperLeader) {
1242 DEBUG(dbgs() << "Store leader changed, fixing\n");
1243 EClass->RepLeader = ProperLeader;
1244 markLeaderChangeTouched(EClass);
1245 markMemoryUsersTouched(MSSA->getMemoryAccess(SI));
1250 // Process the fact that Edge (from, to) is reachable, including marking
1251 // any newly reachable blocks and instructions for processing.
1252 void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
1253 // Check if the Edge was reachable before.
1254 if (ReachableEdges.insert({From, To}).second) {
1255 // If this block wasn't reachable before, all instructions are touched.
1256 if (ReachableBlocks.insert(To).second) {
1257 DEBUG(dbgs() << "Block " << getBlockName(To) << " marked reachable\n");
1258 const auto &InstRange = BlockInstRange.lookup(To);
1259 TouchedInstructions.set(InstRange.first, InstRange.second);
1261 DEBUG(dbgs() << "Block " << getBlockName(To)
1262 << " was reachable, but new edge {" << getBlockName(From)
1263 << "," << getBlockName(To) << "} to it found\n");
1265 // We've made an edge reachable to an existing block, which may
1266 // impact predicates. Otherwise, only mark the phi nodes as touched, as
1267 // they are the only thing that depend on new edges. Anything using their
1268 // values will get propagated to if necessary.
1269 if (MemoryAccess *MemPhi = MSSA->getMemoryAccess(To))
1270 TouchedInstructions.set(InstrDFS[MemPhi]);
1272 auto BI = To->begin();
1273 while (isa<PHINode>(BI)) {
1274 TouchedInstructions.set(InstrDFS[&*BI]);
1281 // Given a predicate condition (from a switch, cmp, or whatever) and a block,
1282 // see if we know some constant value for it already.
1283 Value *NewGVN::findConditionEquivalence(Value *Cond, BasicBlock *B) const {
1284 auto Result = lookupOperandLeader(Cond, nullptr, B);
1285 if (isa<Constant>(Result))
1290 // Process the outgoing edges of a block for reachability.
1291 void NewGVN::processOutgoingEdges(TerminatorInst *TI, BasicBlock *B) {
1292 // Evaluate reachability of terminator instruction.
1294 if ((BR = dyn_cast<BranchInst>(TI)) && BR->isConditional()) {
1295 Value *Cond = BR->getCondition();
1296 Value *CondEvaluated = findConditionEquivalence(Cond, B);
1297 if (!CondEvaluated) {
1298 if (auto *I = dyn_cast<Instruction>(Cond)) {
1299 const Expression *E = createExpression(I, B);
1300 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
1301 CondEvaluated = CE->getConstantValue();
1303 } else if (isa<ConstantInt>(Cond)) {
1304 CondEvaluated = Cond;
1308 BasicBlock *TrueSucc = BR->getSuccessor(0);
1309 BasicBlock *FalseSucc = BR->getSuccessor(1);
1310 if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) {
1312 DEBUG(dbgs() << "Condition for Terminator " << *TI
1313 << " evaluated to true\n");
1314 updateReachableEdge(B, TrueSucc);
1315 } else if (CI->isZero()) {
1316 DEBUG(dbgs() << "Condition for Terminator " << *TI
1317 << " evaluated to false\n");
1318 updateReachableEdge(B, FalseSucc);
1321 updateReachableEdge(B, TrueSucc);
1322 updateReachableEdge(B, FalseSucc);
1324 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
1325 // For switches, propagate the case values into the case
1328 // Remember how many outgoing edges there are to every successor.
1329 SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges;
1331 Value *SwitchCond = SI->getCondition();
1332 Value *CondEvaluated = findConditionEquivalence(SwitchCond, B);
1333 // See if we were able to turn this switch statement into a constant.
1334 if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) {
1335 auto *CondVal = cast<ConstantInt>(CondEvaluated);
1336 // We should be able to get case value for this.
1337 auto CaseVal = SI->findCaseValue(CondVal);
1338 if (CaseVal.getCaseSuccessor() == SI->getDefaultDest()) {
1339 // We proved the value is outside of the range of the case.
1340 // We can't do anything other than mark the default dest as reachable,
1342 updateReachableEdge(B, SI->getDefaultDest());
1345 // Now get where it goes and mark it reachable.
1346 BasicBlock *TargetBlock = CaseVal.getCaseSuccessor();
1347 updateReachableEdge(B, TargetBlock);
1349 for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; ++i) {
1350 BasicBlock *TargetBlock = SI->getSuccessor(i);
1351 ++SwitchEdges[TargetBlock];
1352 updateReachableEdge(B, TargetBlock);
1356 // Otherwise this is either unconditional, or a type we have no
1357 // idea about. Just mark successors as reachable.
1358 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
1359 BasicBlock *TargetBlock = TI->getSuccessor(i);
1360 updateReachableEdge(B, TargetBlock);
1363 // This also may be a memory defining terminator, in which case, set it
1364 // equivalent to nothing.
1365 if (MemoryAccess *MA = MSSA->getMemoryAccess(TI))
1366 setMemoryAccessEquivTo(MA, nullptr);
1370 // The algorithm initially places the values of the routine in the INITIAL
1372 // class. The leader of INITIAL is the undetermined value `TOP`.
1373 // When the algorithm has finished, values still in INITIAL are unreachable.
1374 void NewGVN::initializeCongruenceClasses(Function &F) {
1375 // FIXME now i can't remember why this is 2
1376 NextCongruenceNum = 2;
1377 // Initialize all other instructions to be in INITIAL class.
1378 CongruenceClass::MemberSet InitialValues;
1379 InitialClass = createCongruenceClass(nullptr, nullptr);
1381 if (auto *MP = MSSA->getMemoryAccess(&B))
1382 MemoryAccessEquiv.insert({MP, MSSA->getLiveOnEntryDef()});
1385 InitialValues.insert(&I);
1386 ValueToClass[&I] = InitialClass;
1387 // All memory accesses are equivalent to live on entry to start. They must
1388 // be initialized to something so that initial changes are noticed. For
1389 // the maximal answer, we initialize them all to be the same as
1390 // liveOnEntry. Note that to save time, we only initialize the
1391 // MemoryDef's for stores and all MemoryPhis to be equal. Right now, no
1392 // other expression can generate a memory equivalence. If we start
1393 // handling memcpy/etc, we can expand this.
1394 if (isa<StoreInst>(&I)) {
1395 MemoryAccessEquiv.insert(
1396 {MSSA->getMemoryAccess(&I), MSSA->getLiveOnEntryDef()});
1397 ++InitialClass->StoreCount;
1398 assert(InitialClass->StoreCount > 0);
1402 InitialClass->Members.swap(InitialValues);
1404 // Initialize arguments to be in their own unique congruence classes
1405 for (auto &FA : F.args())
1406 createSingletonCongruenceClass(&FA);
1409 void NewGVN::cleanupTables() {
1410 for (unsigned i = 0, e = CongruenceClasses.size(); i != e; ++i) {
1411 DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->ID << " has "
1412 << CongruenceClasses[i]->Members.size() << " members\n");
1413 // Make sure we delete the congruence class (probably worth switching to
1414 // a unique_ptr at some point.
1415 delete CongruenceClasses[i];
1416 CongruenceClasses[i] = nullptr;
1419 ValueToClass.clear();
1420 ArgRecycler.clear(ExpressionAllocator);
1421 ExpressionAllocator.Reset();
1422 CongruenceClasses.clear();
1423 ExpressionToClass.clear();
1424 ValueToExpression.clear();
1425 ReachableBlocks.clear();
1426 ReachableEdges.clear();
1428 ProcessedCount.clear();
1432 InstructionsToErase.clear();
1435 BlockInstRange.clear();
1436 TouchedInstructions.clear();
1437 DominatedInstRange.clear();
1438 MemoryAccessEquiv.clear();
1441 std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
1443 unsigned End = Start;
1444 if (MemoryAccess *MemPhi = MSSA->getMemoryAccess(B)) {
1445 InstrDFS[MemPhi] = End++;
1446 DFSToInstr.emplace_back(MemPhi);
1449 for (auto &I : *B) {
1450 InstrDFS[&I] = End++;
1451 DFSToInstr.emplace_back(&I);
1454 // All of the range functions taken half-open ranges (open on the end side).
1455 // So we do not subtract one from count, because at this point it is one
1456 // greater than the last instruction.
1457 return std::make_pair(Start, End);
1460 void NewGVN::updateProcessedCount(Value *V) {
1462 if (ProcessedCount.count(V) == 0) {
1463 ProcessedCount.insert({V, 1});
1465 ProcessedCount[V] += 1;
1466 assert(ProcessedCount[V] < 100 &&
1467 "Seem to have processed the same Value a lot");
1471 // Evaluate MemoryPhi nodes symbolically, just like PHI nodes
1472 void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
1473 // If all the arguments are the same, the MemoryPhi has the same value as the
1475 // Filter out unreachable blocks from our operands.
1476 auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {
1477 return ReachableBlocks.count(MP->getIncomingBlock(U));
1480 assert(Filtered.begin() != Filtered.end() &&
1481 "We should not be processing a MemoryPhi in a completely "
1482 "unreachable block");
1484 // Transform the remaining operands into operand leaders.
1485 // FIXME: mapped_iterator should have a range version.
1486 auto LookupFunc = [&](const Use &U) {
1487 return lookupMemoryAccessEquiv(cast<MemoryAccess>(U));
1489 auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);
1490 auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);
1492 // and now check if all the elements are equal.
1493 // Sadly, we can't use std::equals since these are random access iterators.
1494 MemoryAccess *AllSameValue = *MappedBegin;
1496 bool AllEqual = std::all_of(
1497 MappedBegin, MappedEnd,
1498 [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });
1501 DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue << "\n");
1503 DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
1505 if (setMemoryAccessEquivTo(MP, AllEqual ? AllSameValue : nullptr))
1506 markMemoryUsersTouched(MP);
1509 // Value number a single instruction, symbolically evaluating, performing
1510 // congruence finding, and updating mappings.
1511 void NewGVN::valueNumberInstruction(Instruction *I) {
1512 DEBUG(dbgs() << "Processing instruction " << *I << "\n");
1513 if (isInstructionTriviallyDead(I, TLI)) {
1514 DEBUG(dbgs() << "Skipping unused instruction\n");
1515 markInstructionForDeletion(I);
1518 if (!I->isTerminator()) {
1519 const auto *Symbolized = performSymbolicEvaluation(I, I->getParent());
1520 // If we couldn't come up with a symbolic expression, use the unknown
1522 if (Symbolized == nullptr)
1523 Symbolized = createUnknownExpression(I);
1524 performCongruenceFinding(I, Symbolized);
1526 // Handle terminators that return values. All of them produce values we
1527 // don't currently understand.
1528 if (!I->getType()->isVoidTy()) {
1529 auto *Symbolized = createUnknownExpression(I);
1530 performCongruenceFinding(I, Symbolized);
1532 processOutgoingEdges(dyn_cast<TerminatorInst>(I), I->getParent());
1536 // Check if there is a path, using single or equal argument phi nodes, from
1538 bool NewGVN::singleReachablePHIPath(const MemoryAccess *First,
1539 const MemoryAccess *Second) const {
1540 if (First == Second)
1543 if (auto *FirstDef = dyn_cast<MemoryUseOrDef>(First)) {
1544 auto *DefAccess = FirstDef->getDefiningAccess();
1545 return singleReachablePHIPath(DefAccess, Second);
1547 auto *MP = cast<MemoryPhi>(First);
1548 auto ReachableOperandPred = [&](const Use &U) {
1549 return ReachableBlocks.count(MP->getIncomingBlock(U));
1551 auto FilteredPhiArgs =
1552 make_filter_range(MP->operands(), ReachableOperandPred);
1553 SmallVector<const Value *, 32> OperandList;
1554 std::copy(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
1555 std::back_inserter(OperandList));
1556 bool Okay = OperandList.size() == 1;
1558 Okay = std::equal(OperandList.begin(), OperandList.end(),
1559 OperandList.begin());
1561 return singleReachablePHIPath(cast<MemoryAccess>(OperandList[0]), Second);
1566 // Verify the that the memory equivalence table makes sense relative to the
1567 // congruence classes. Note that this checking is not perfect, and is currently
1568 // subject to very rare false negatives. It is only useful for testing/debugging.
1569 void NewGVN::verifyMemoryCongruency() const {
1570 // Anything equivalent in the memory access table should be in the same
1571 // congruence class.
1573 // Filter out the unreachable and trivially dead entries, because they may
1574 // never have been updated if the instructions were not processed.
1575 auto ReachableAccessPred =
1576 [&](const std::pair<const MemoryAccess *, MemoryAccess *> Pair) {
1577 bool Result = ReachableBlocks.count(Pair.first->getBlock());
1580 if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
1581 return !isInstructionTriviallyDead(MemDef->getMemoryInst());
1585 auto Filtered = make_filter_range(MemoryAccessEquiv, ReachableAccessPred);
1586 for (auto KV : Filtered) {
1587 assert(KV.first != KV.second &&
1588 "We added a useless equivalence to the memory equivalence table");
1589 // Unreachable instructions may not have changed because we never process
1591 if (!ReachableBlocks.count(KV.first->getBlock()))
1593 if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
1594 auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second);
1595 if (FirstMUD && SecondMUD)
1596 assert((singleReachablePHIPath(FirstMUD, SecondMUD) ||
1597 ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
1598 ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
1599 "The instructions for these memory operations should have "
1600 "been in the same congruence class or reachable through"
1601 "a single argument phi");
1602 } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
1604 // We can only sanely verify that MemoryDefs in the operand list all have
1606 auto ReachableOperandPred = [&](const Use &U) {
1607 return ReachableBlocks.count(FirstMP->getIncomingBlock(U)) &&
1611 // All arguments should in the same class, ignoring unreachable arguments
1612 auto FilteredPhiArgs =
1613 make_filter_range(FirstMP->operands(), ReachableOperandPred);
1614 SmallVector<const CongruenceClass *, 16> PhiOpClasses;
1615 std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
1616 std::back_inserter(PhiOpClasses), [&](const Use &U) {
1617 const MemoryDef *MD = cast<MemoryDef>(U);
1618 return ValueToClass.lookup(MD->getMemoryInst());
1620 assert(std::equal(PhiOpClasses.begin(), PhiOpClasses.end(),
1621 PhiOpClasses.begin()) &&
1622 "All MemoryPhi arguments should be in the same class");
1627 // This is the main transformation entry point.
1628 bool NewGVN::runGVN(Function &F, DominatorTree *_DT, AssumptionCache *_AC,
1629 TargetLibraryInfo *_TLI, AliasAnalysis *_AA,
1631 bool Changed = false;
1637 DL = &F.getParent()->getDataLayout();
1638 MSSAWalker = MSSA->getWalker();
1640 // Count number of instructions for sizing of hash tables, and come
1641 // up with a global dfs numbering for instructions.
1642 unsigned ICount = 1;
1643 // Add an empty instruction to account for the fact that we start at 1
1644 DFSToInstr.emplace_back(nullptr);
1645 // Note: We want RPO traversal of the blocks, which is not quite the same as
1646 // dominator tree order, particularly with regard whether backedges get
1647 // visited first or second, given a block with multiple successors.
1648 // If we visit in the wrong order, we will end up performing N times as many
1650 // The dominator tree does guarantee that, for a given dom tree node, it's
1651 // parent must occur before it in the RPO ordering. Thus, we only need to sort
1653 DenseMap<const DomTreeNode *, unsigned> RPOOrdering;
1654 ReversePostOrderTraversal<Function *> RPOT(&F);
1655 unsigned Counter = 0;
1656 for (auto &B : RPOT) {
1657 auto *Node = DT->getNode(B);
1658 assert(Node && "RPO and Dominator tree should have same reachability");
1659 RPOOrdering[Node] = ++Counter;
1661 // Sort dominator tree children arrays into RPO.
1662 for (auto &B : RPOT) {
1663 auto *Node = DT->getNode(B);
1664 if (Node->getChildren().size() > 1)
1665 std::sort(Node->begin(), Node->end(),
1666 [&RPOOrdering](const DomTreeNode *A, const DomTreeNode *B) {
1667 return RPOOrdering[A] < RPOOrdering[B];
1671 // Now a standard depth first ordering of the domtree is equivalent to RPO.
1672 auto DFI = df_begin(DT->getRootNode());
1673 for (auto DFE = df_end(DT->getRootNode()); DFI != DFE; ++DFI) {
1674 BasicBlock *B = DFI->getBlock();
1675 const auto &BlockRange = assignDFSNumbers(B, ICount);
1676 BlockInstRange.insert({B, BlockRange});
1677 ICount += BlockRange.second - BlockRange.first;
1680 // Handle forward unreachable blocks and figure out which blocks
1681 // have single preds.
1683 // Assign numbers to unreachable blocks.
1684 if (!DFI.nodeVisited(DT->getNode(&B))) {
1685 const auto &BlockRange = assignDFSNumbers(&B, ICount);
1686 BlockInstRange.insert({&B, BlockRange});
1687 ICount += BlockRange.second - BlockRange.first;
1691 TouchedInstructions.resize(ICount);
1692 DominatedInstRange.reserve(F.size());
1693 // Ensure we don't end up resizing the expressionToClass map, as
1694 // that can be quite expensive. At most, we have one expression per
1696 ExpressionToClass.reserve(ICount);
1698 // Initialize the touched instructions to include the entry block.
1699 const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());
1700 TouchedInstructions.set(InstRange.first, InstRange.second);
1701 ReachableBlocks.insert(&F.getEntryBlock());
1703 initializeCongruenceClasses(F);
1705 unsigned int Iterations = 0;
1706 // We start out in the entry block.
1707 BasicBlock *LastBlock = &F.getEntryBlock();
1708 while (TouchedInstructions.any()) {
1710 // Walk through all the instructions in all the blocks in RPO.
1711 for (int InstrNum = TouchedInstructions.find_first(); InstrNum != -1;
1712 InstrNum = TouchedInstructions.find_next(InstrNum)) {
1713 assert(InstrNum != 0 && "Bit 0 should never be set, something touched an "
1714 "instruction not in the lookup table");
1715 Value *V = DFSToInstr[InstrNum];
1716 BasicBlock *CurrBlock = nullptr;
1718 if (auto *I = dyn_cast<Instruction>(V))
1719 CurrBlock = I->getParent();
1720 else if (auto *MP = dyn_cast<MemoryPhi>(V))
1721 CurrBlock = MP->getBlock();
1723 llvm_unreachable("DFSToInstr gave us an unknown type of instruction");
1725 // If we hit a new block, do reachability processing.
1726 if (CurrBlock != LastBlock) {
1727 LastBlock = CurrBlock;
1728 bool BlockReachable = ReachableBlocks.count(CurrBlock);
1729 const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);
1731 // If it's not reachable, erase any touched instructions and move on.
1732 if (!BlockReachable) {
1733 TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);
1734 DEBUG(dbgs() << "Skipping instructions in block "
1735 << getBlockName(CurrBlock)
1736 << " because it is unreachable\n");
1739 updateProcessedCount(CurrBlock);
1742 if (auto *MP = dyn_cast<MemoryPhi>(V)) {
1743 DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
1744 valueNumberMemoryPhi(MP);
1745 } else if (auto *I = dyn_cast<Instruction>(V)) {
1746 valueNumberInstruction(I);
1748 llvm_unreachable("Should have been a MemoryPhi or Instruction");
1750 updateProcessedCount(V);
1751 // Reset after processing (because we may mark ourselves as touched when
1752 // we propagate equalities).
1753 TouchedInstructions.reset(InstrNum);
1756 NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);
1758 verifyMemoryCongruency();
1760 Changed |= eliminateInstructions(F);
1762 // Delete all instructions marked for deletion.
1763 for (Instruction *ToErase : InstructionsToErase) {
1764 if (!ToErase->use_empty())
1765 ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType()));
1767 ToErase->eraseFromParent();
1770 // Delete all unreachable blocks.
1771 auto UnreachableBlockPred = [&](const BasicBlock &BB) {
1772 return !ReachableBlocks.count(&BB);
1775 for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {
1776 DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
1777 << " is unreachable\n");
1778 deleteInstructionsInBlock(&BB);
1786 bool NewGVN::runOnFunction(Function &F) {
1787 if (skipFunction(F))
1789 return runGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
1790 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
1791 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
1792 &getAnalysis<AAResultsWrapperPass>().getAAResults(),
1793 &getAnalysis<MemorySSAWrapperPass>().getMSSA());
1796 PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
1799 // Apparently the order in which we get these results matter for
1800 // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
1801 // the same order here, just in case.
1802 auto &AC = AM.getResult<AssumptionAnalysis>(F);
1803 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
1804 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
1805 auto &AA = AM.getResult<AAManager>(F);
1806 auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
1807 bool Changed = Impl.runGVN(F, &DT, &AC, &TLI, &AA, &MSSA);
1809 return PreservedAnalyses::all();
1810 PreservedAnalyses PA;
1811 PA.preserve<DominatorTreeAnalysis>();
1812 PA.preserve<GlobalsAA>();
1816 // Return true if V is a value that will always be available (IE can
1817 // be placed anywhere) in the function. We don't do globals here
1818 // because they are often worse to put in place.
1819 // TODO: Separate cost from availability
1820 static bool alwaysAvailable(Value *V) {
1821 return isa<Constant>(V) || isa<Argument>(V);
1824 // Get the basic block from an instruction/value.
1825 static BasicBlock *getBlockForValue(Value *V) {
1826 if (auto *I = dyn_cast<Instruction>(V))
1827 return I->getParent();
1831 struct NewGVN::ValueDFS {
1835 // Only one of these will be set.
1836 Value *Val = nullptr;
1839 bool operator<(const ValueDFS &Other) const {
1840 // It's not enough that any given field be less than - we have sets
1841 // of fields that need to be evaluated together to give a proper ordering.
1842 // For example, if you have;
1847 // We want the second to be less than the first, but if we just go field
1848 // by field, we will get to Val 0 < Val 50 and say the first is less than
1849 // the second. We only want it to be less than if the DFS orders are equal.
1851 // Each LLVM instruction only produces one value, and thus the lowest-level
1852 // differentiator that really matters for the stack (and what we use as as a
1853 // replacement) is the local dfs number.
1854 // Everything else in the structure is instruction level, and only affects
1855 // the order in which we will replace operands of a given instruction.
1857 // For a given instruction (IE things with equal dfsin, dfsout, localnum),
1858 // the order of replacement of uses does not matter.
1862 // When you hit b, you will have two valuedfs with the same dfsin, out, and
1864 // The .val will be the same as well.
1865 // The .u's will be different.
1866 // You will replace both, and it does not matter what order you replace them
1867 // in (IE whether you replace operand 2, then operand 1, or operand 1, then
1869 // Similarly for the case of same dfsin, dfsout, localnum, but different
1874 // in c, we will a valuedfs for a, and one for b,with everything the same
1876 // It does not matter what order we replace these operands in.
1877 // You will always end up with the same IR, and this is guaranteed.
1878 return std::tie(DFSIn, DFSOut, LocalNum, Val, U) <
1879 std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Val,
1884 void NewGVN::convertDenseToDFSOrdered(
1885 CongruenceClass::MemberSet &Dense,
1886 SmallVectorImpl<ValueDFS> &DFSOrderedSet) {
1887 for (auto D : Dense) {
1888 // First add the value.
1889 BasicBlock *BB = getBlockForValue(D);
1890 // Constants are handled prior to ever calling this function, so
1891 // we should only be left with instructions as members.
1892 assert(BB && "Should have figured out a basic block for value");
1895 std::pair<int, int> DFSPair = DFSDomMap[BB];
1896 assert(DFSPair.first != -1 && DFSPair.second != -1 && "Invalid DFS Pair");
1897 VD.DFSIn = DFSPair.first;
1898 VD.DFSOut = DFSPair.second;
1900 // If it's an instruction, use the real local dfs number.
1901 if (auto *I = dyn_cast<Instruction>(D))
1902 VD.LocalNum = InstrDFS[I];
1904 llvm_unreachable("Should have been an instruction");
1906 DFSOrderedSet.emplace_back(VD);
1908 // Now add the users.
1909 for (auto &U : D->uses()) {
1910 if (auto *I = dyn_cast<Instruction>(U.getUser())) {
1912 // Put the phi node uses in the incoming block.
1914 if (auto *P = dyn_cast<PHINode>(I)) {
1915 IBlock = P->getIncomingBlock(U);
1916 // Make phi node users appear last in the incoming block
1918 VD.LocalNum = InstrDFS.size() + 1;
1920 IBlock = I->getParent();
1921 VD.LocalNum = InstrDFS[I];
1923 std::pair<int, int> DFSPair = DFSDomMap[IBlock];
1924 VD.DFSIn = DFSPair.first;
1925 VD.DFSOut = DFSPair.second;
1927 DFSOrderedSet.emplace_back(VD);
1933 static void patchReplacementInstruction(Instruction *I, Value *Repl) {
1934 // Patch the replacement so that it is not more restrictive than the value
1936 auto *Op = dyn_cast<BinaryOperator>(I);
1937 auto *ReplOp = dyn_cast<BinaryOperator>(Repl);
1940 ReplOp->andIRFlags(Op);
1942 if (auto *ReplInst = dyn_cast<Instruction>(Repl)) {
1943 // FIXME: If both the original and replacement value are part of the
1944 // same control-flow region (meaning that the execution of one
1945 // guarentees the executation of the other), then we can combine the
1946 // noalias scopes here and do better than the general conservative
1947 // answer used in combineMetadata().
1949 // In general, GVN unifies expressions over different control-flow
1950 // regions, and so we need a conservative combination of the noalias
1952 unsigned KnownIDs[] = {
1953 LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
1954 LLVMContext::MD_noalias, LLVMContext::MD_range,
1955 LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load,
1956 LLVMContext::MD_invariant_group};
1957 combineMetadata(ReplInst, I, KnownIDs);
1961 static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
1962 patchReplacementInstruction(I, Repl);
1963 I->replaceAllUsesWith(Repl);
1966 void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
1967 DEBUG(dbgs() << " BasicBlock Dead:" << *BB);
1968 ++NumGVNBlocksDeleted;
1970 // Check to see if there are non-terminating instructions to delete.
1971 if (isa<TerminatorInst>(BB->begin()))
1974 // Delete the instructions backwards, as it has a reduced likelihood of having
1975 // to update as many def-use and use-def chains. Start after the terminator.
1976 auto StartPoint = BB->rbegin();
1978 // Note that we explicitly recalculate BB->rend() on each iteration,
1979 // as it may change when we remove the first instruction.
1980 for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
1981 Instruction &Inst = *I++;
1982 if (!Inst.use_empty())
1983 Inst.replaceAllUsesWith(UndefValue::get(Inst.getType()));
1984 if (isa<LandingPadInst>(Inst))
1987 Inst.eraseFromParent();
1988 ++NumGVNInstrDeleted;
1992 void NewGVN::markInstructionForDeletion(Instruction *I) {
1993 DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
1994 InstructionsToErase.insert(I);
1997 void NewGVN::replaceInstruction(Instruction *I, Value *V) {
1999 DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
2000 patchAndReplaceAllUsesWith(I, V);
2001 // We save the actual erasing to avoid invalidating memory
2002 // dependencies until we are done with everything.
2003 markInstructionForDeletion(I);
2008 // This is a stack that contains both the value and dfs info of where
2009 // that value is valid.
2010 class ValueDFSStack {
2012 Value *back() const { return ValueStack.back(); }
2013 std::pair<int, int> dfs_back() const { return DFSStack.back(); }
2015 void push_back(Value *V, int DFSIn, int DFSOut) {
2016 ValueStack.emplace_back(V);
2017 DFSStack.emplace_back(DFSIn, DFSOut);
2019 bool empty() const { return DFSStack.empty(); }
2020 bool isInScope(int DFSIn, int DFSOut) const {
2023 return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
2026 void popUntilDFSScope(int DFSIn, int DFSOut) {
2028 // These two should always be in sync at this point.
2029 assert(ValueStack.size() == DFSStack.size() &&
2030 "Mismatch between ValueStack and DFSStack");
2032 !DFSStack.empty() &&
2033 !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {
2034 DFSStack.pop_back();
2035 ValueStack.pop_back();
2040 SmallVector<Value *, 8> ValueStack;
2041 SmallVector<std::pair<int, int>, 8> DFSStack;
2045 bool NewGVN::eliminateInstructions(Function &F) {
2046 // This is a non-standard eliminator. The normal way to eliminate is
2047 // to walk the dominator tree in order, keeping track of available
2048 // values, and eliminating them. However, this is mildly
2049 // pointless. It requires doing lookups on every instruction,
2050 // regardless of whether we will ever eliminate it. For
2051 // instructions part of most singleton congruence classes, we know we
2052 // will never eliminate them.
2054 // Instead, this eliminator looks at the congruence classes directly, sorts
2055 // them into a DFS ordering of the dominator tree, and then we just
2056 // perform elimination straight on the sets by walking the congruence
2057 // class member uses in order, and eliminate the ones dominated by the
2058 // last member. This is worst case O(E log E) where E = number of
2059 // instructions in a single congruence class. In theory, this is all
2060 // instructions. In practice, it is much faster, as most instructions are
2061 // either in singleton congruence classes or can't possibly be eliminated
2062 // anyway (if there are no overlapping DFS ranges in class).
2063 // When we find something not dominated, it becomes the new leader
2064 // for elimination purposes.
2065 // TODO: If we wanted to be faster, We could remove any members with no
2066 // overlapping ranges while sorting, as we will never eliminate anything
2067 // with those members, as they don't dominate anything else in our set.
2069 bool AnythingReplaced = false;
2071 // Since we are going to walk the domtree anyway, and we can't guarantee the
2072 // DFS numbers are updated, we compute some ourselves.
2073 DT->updateDFSNumbers();
2076 if (!ReachableBlocks.count(&B)) {
2077 for (const auto S : successors(&B)) {
2078 for (auto II = S->begin(); isa<PHINode>(II); ++II) {
2079 auto &Phi = cast<PHINode>(*II);
2080 DEBUG(dbgs() << "Replacing incoming value of " << *II << " for block "
2082 << " with undef due to it being unreachable\n");
2083 for (auto &Operand : Phi.incoming_values())
2084 if (Phi.getIncomingBlock(Operand) == &B)
2085 Operand.set(UndefValue::get(Phi.getType()));
2089 DomTreeNode *Node = DT->getNode(&B);
2091 DFSDomMap[&B] = {Node->getDFSNumIn(), Node->getDFSNumOut()};
2094 for (CongruenceClass *CC : CongruenceClasses) {
2095 // FIXME: We should eventually be able to replace everything still
2096 // in the initial class with undef, as they should be unreachable.
2097 // Right now, initial still contains some things we skip value
2098 // numbering of (UNREACHABLE's, for example).
2099 if (CC == InitialClass || CC->Dead)
2101 assert(CC->RepLeader && "We should have had a leader");
2103 // If this is a leader that is always available, and it's a
2104 // constant or has no equivalences, just replace everything with
2105 // it. We then update the congruence class with whatever members
2107 if (alwaysAvailable(CC->RepLeader)) {
2108 SmallPtrSet<Value *, 4> MembersLeft;
2109 for (auto M : CC->Members) {
2113 // Void things have no uses we can replace.
2114 if (Member == CC->RepLeader || Member->getType()->isVoidTy()) {
2115 MembersLeft.insert(Member);
2119 DEBUG(dbgs() << "Found replacement " << *(CC->RepLeader) << " for "
2120 << *Member << "\n");
2121 // Due to equality propagation, these may not always be
2122 // instructions, they may be real values. We don't really
2123 // care about trying to replace the non-instructions.
2124 if (auto *I = dyn_cast<Instruction>(Member)) {
2125 assert(CC->RepLeader != I &&
2126 "About to accidentally remove our leader");
2127 replaceInstruction(I, CC->RepLeader);
2128 AnythingReplaced = true;
2132 MembersLeft.insert(I);
2135 CC->Members.swap(MembersLeft);
2138 DEBUG(dbgs() << "Eliminating in congruence class " << CC->ID << "\n");
2139 // If this is a singleton, we can skip it.
2140 if (CC->Members.size() != 1) {
2142 // This is a stack because equality replacement/etc may place
2143 // constants in the middle of the member list, and we want to use
2144 // those constant values in preference to the current leader, over
2145 // the scope of those constants.
2146 ValueDFSStack EliminationStack;
2148 // Convert the members to DFS ordered sets and then merge them.
2149 SmallVector<ValueDFS, 8> DFSOrderedSet;
2150 convertDenseToDFSOrdered(CC->Members, DFSOrderedSet);
2152 // Sort the whole thing.
2153 std::sort(DFSOrderedSet.begin(), DFSOrderedSet.end());
2155 for (auto &VD : DFSOrderedSet) {
2156 int MemberDFSIn = VD.DFSIn;
2157 int MemberDFSOut = VD.DFSOut;
2158 Value *Member = VD.Val;
2159 Use *MemberUse = VD.U;
2162 // We ignore void things because we can't get a value from them.
2163 // FIXME: We could actually use this to kill dead stores that are
2164 // dominated by equivalent earlier stores.
2165 if (Member->getType()->isVoidTy())
2169 if (EliminationStack.empty()) {
2170 DEBUG(dbgs() << "Elimination Stack is empty\n");
2172 DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
2173 << EliminationStack.dfs_back().first << ","
2174 << EliminationStack.dfs_back().second << ")\n");
2177 DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
2178 << MemberDFSOut << ")\n");
2179 // First, we see if we are out of scope or empty. If so,
2180 // and there equivalences, we try to replace the top of
2181 // stack with equivalences (if it's on the stack, it must
2182 // not have been eliminated yet).
2183 // Then we synchronize to our current scope, by
2184 // popping until we are back within a DFS scope that
2185 // dominates the current member.
2186 // Then, what happens depends on a few factors
2187 // If the stack is now empty, we need to push
2188 // If we have a constant or a local equivalence we want to
2189 // start using, we also push.
2190 // Otherwise, we walk along, processing members who are
2191 // dominated by this scope, and eliminate them.
2193 Member && (EliminationStack.empty() || isa<Constant>(Member));
2195 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);
2197 if (OutOfScope || ShouldPush) {
2198 // Sync to our current scope.
2199 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
2200 ShouldPush |= Member && EliminationStack.empty();
2202 EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);
2206 // If we get to this point, and the stack is empty we must have a use
2207 // with nothing we can use to eliminate it, just skip it.
2208 if (EliminationStack.empty())
2211 // Skip the Value's, we only want to eliminate on their uses.
2214 Value *Result = EliminationStack.back();
2216 // Don't replace our existing users with ourselves.
2217 if (MemberUse->get() == Result)
2220 DEBUG(dbgs() << "Found replacement " << *Result << " for "
2221 << *MemberUse->get() << " in " << *(MemberUse->getUser())
2224 // If we replaced something in an instruction, handle the patching of
2226 if (auto *ReplacedInst = dyn_cast<Instruction>(MemberUse->get()))
2227 patchReplacementInstruction(ReplacedInst, Result);
2229 assert(isa<Instruction>(MemberUse->getUser()));
2230 MemberUse->set(Result);
2231 AnythingReplaced = true;
2236 // Cleanup the congruence class.
2237 SmallPtrSet<Value *, 4> MembersLeft;
2238 for (Value *Member : CC->Members) {
2239 if (Member->getType()->isVoidTy()) {
2240 MembersLeft.insert(Member);
2244 if (auto *MemberInst = dyn_cast<Instruction>(Member)) {
2245 if (isInstructionTriviallyDead(MemberInst)) {
2246 // TODO: Don't mark loads of undefs.
2247 markInstructionForDeletion(MemberInst);
2251 MembersLeft.insert(Member);
2253 CC->Members.swap(MembersLeft);
2256 return AnythingReplaced;