1 //===- AMDGPUUnifyDivergentExitNodes.cpp ----------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This is a variant of the UnifyDivergentExitNodes pass. Rather than ensuring
11 // there is at most one ret and one unreachable instruction, it ensures there is
12 // at most one divergent exiting block.
14 // StructurizeCFG can't deal with multi-exit regions formed by branches to
15 // multiple return nodes. It is not desirable to structurize regions with
16 // uniform branches, so unifying those to the same return block as divergent
17 // branches inhibits use of scalar branching. It still can't deal with the case
18 // where one branch goes to return, and one unreachable. Replace unreachable in
19 // this case with a return.
21 //===----------------------------------------------------------------------===//
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallPtrSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/Analysis/DivergenceAnalysis.h"
29 #include "llvm/Analysis/PostDominators.h"
30 #include "llvm/Analysis/TargetTransformInfo.h"
31 #include "llvm/IR/BasicBlock.h"
32 #include "llvm/IR/CFG.h"
33 #include "llvm/IR/Constants.h"
34 #include "llvm/IR/Function.h"
35 #include "llvm/IR/InstrTypes.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/Intrinsics.h"
38 #include "llvm/IR/Type.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/Casting.h"
41 #include "llvm/Transforms/Scalar.h"
42 #include "llvm/Transforms/Utils/Local.h"
46 #define DEBUG_TYPE "amdgpu-unify-divergent-exit-nodes"
50 class AMDGPUUnifyDivergentExitNodes : public FunctionPass {
52 static char ID; // Pass identification, replacement for typeid
54 AMDGPUUnifyDivergentExitNodes() : FunctionPass(ID) {
55 initializeAMDGPUUnifyDivergentExitNodesPass(*PassRegistry::getPassRegistry());
58 // We can preserve non-critical-edgeness when we unify function exit nodes
59 void getAnalysisUsage(AnalysisUsage &AU) const override;
60 bool runOnFunction(Function &F) override;
63 } // end anonymous namespace
65 char AMDGPUUnifyDivergentExitNodes::ID = 0;
67 char &llvm::AMDGPUUnifyDivergentExitNodesID = AMDGPUUnifyDivergentExitNodes::ID;
69 INITIALIZE_PASS_BEGIN(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE,
70 "Unify divergent function exit nodes", false, false)
71 INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
72 INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
73 INITIALIZE_PASS_END(AMDGPUUnifyDivergentExitNodes, DEBUG_TYPE,
74 "Unify divergent function exit nodes", false, false)
76 void AMDGPUUnifyDivergentExitNodes::getAnalysisUsage(AnalysisUsage &AU) const{
77 // TODO: Preserve dominator tree.
78 AU.addRequired<PostDominatorTreeWrapperPass>();
80 AU.addRequired<DivergenceAnalysis>();
82 // No divergent values are changed, only blocks and branch edges.
83 AU.addPreserved<DivergenceAnalysis>();
85 // We preserve the non-critical-edgeness property
86 AU.addPreservedID(BreakCriticalEdgesID);
88 // This is a cluster of orthogonal Transforms
89 AU.addPreservedID(LowerSwitchID);
90 FunctionPass::getAnalysisUsage(AU);
92 AU.addRequired<TargetTransformInfoWrapperPass>();
95 /// \returns true if \p BB is reachable through only uniform branches.
96 /// XXX - Is there a more efficient way to find this?
97 static bool isUniformlyReached(const DivergenceAnalysis &DA,
99 SmallVector<BasicBlock *, 8> Stack;
100 SmallPtrSet<BasicBlock *, 8> Visited;
102 for (BasicBlock *Pred : predecessors(&BB))
103 Stack.push_back(Pred);
105 while (!Stack.empty()) {
106 BasicBlock *Top = Stack.pop_back_val();
107 if (!DA.isUniform(Top->getTerminator()))
110 for (BasicBlock *Pred : predecessors(Top)) {
111 if (Visited.insert(Pred).second)
112 Stack.push_back(Pred);
119 static BasicBlock *unifyReturnBlockSet(Function &F,
120 ArrayRef<BasicBlock *> ReturningBlocks,
121 const TargetTransformInfo &TTI,
123 // Otherwise, we need to insert a new basic block into the function, add a PHI
124 // nodes (if the function returns values), and convert all of the return
125 // instructions into unconditional branches.
126 BasicBlock *NewRetBlock = BasicBlock::Create(F.getContext(), Name, &F);
128 PHINode *PN = nullptr;
129 if (F.getReturnType()->isVoidTy()) {
130 ReturnInst::Create(F.getContext(), nullptr, NewRetBlock);
132 // If the function doesn't return void... add a PHI node to the block...
133 PN = PHINode::Create(F.getReturnType(), ReturningBlocks.size(),
135 NewRetBlock->getInstList().push_back(PN);
136 ReturnInst::Create(F.getContext(), PN, NewRetBlock);
139 // Loop over all of the blocks, replacing the return instruction with an
140 // unconditional branch.
141 for (BasicBlock *BB : ReturningBlocks) {
142 // Add an incoming element to the PHI node for every return instruction that
143 // is merging into this new block...
145 PN->addIncoming(BB->getTerminator()->getOperand(0), BB);
147 BB->getInstList().pop_back(); // Remove the return insn
148 BranchInst::Create(NewRetBlock, BB);
151 for (BasicBlock *BB : ReturningBlocks) {
152 // Cleanup possible branch to unconditional branch to the return.
153 simplifyCFG(BB, TTI, {2});
159 bool AMDGPUUnifyDivergentExitNodes::runOnFunction(Function &F) {
160 auto &PDT = getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
161 if (PDT.getRoots().size() <= 1)
164 DivergenceAnalysis &DA = getAnalysis<DivergenceAnalysis>();
166 // Loop over all of the blocks in a function, tracking all of the blocks that
168 SmallVector<BasicBlock *, 4> ReturningBlocks;
169 SmallVector<BasicBlock *, 4> UnreachableBlocks;
171 for (BasicBlock *BB : PDT.getRoots()) {
172 if (isa<ReturnInst>(BB->getTerminator())) {
173 if (!isUniformlyReached(DA, *BB))
174 ReturningBlocks.push_back(BB);
175 } else if (isa<UnreachableInst>(BB->getTerminator())) {
176 if (!isUniformlyReached(DA, *BB))
177 UnreachableBlocks.push_back(BB);
181 if (!UnreachableBlocks.empty()) {
182 BasicBlock *UnreachableBlock = nullptr;
184 if (UnreachableBlocks.size() == 1) {
185 UnreachableBlock = UnreachableBlocks.front();
187 UnreachableBlock = BasicBlock::Create(F.getContext(),
188 "UnifiedUnreachableBlock", &F);
189 new UnreachableInst(F.getContext(), UnreachableBlock);
191 for (BasicBlock *BB : UnreachableBlocks) {
192 BB->getInstList().pop_back(); // Remove the unreachable inst.
193 BranchInst::Create(UnreachableBlock, BB);
197 if (!ReturningBlocks.empty()) {
198 // Don't create a new unreachable inst if we have a return. The
199 // structurizer/annotator can't handle the multiple exits
201 Type *RetTy = F.getReturnType();
202 Value *RetVal = RetTy->isVoidTy() ? nullptr : UndefValue::get(RetTy);
203 UnreachableBlock->getInstList().pop_back(); // Remove the unreachable inst.
205 Function *UnreachableIntrin =
206 Intrinsic::getDeclaration(F.getParent(), Intrinsic::amdgcn_unreachable);
208 // Insert a call to an intrinsic tracking that this is an unreachable
209 // point, in case we want to kill the active lanes or something later.
210 CallInst::Create(UnreachableIntrin, {}, "", UnreachableBlock);
212 // Don't create a scalar trap. We would only want to trap if this code was
213 // really reached, but a scalar trap would happen even if no lanes
214 // actually reached here.
215 ReturnInst::Create(F.getContext(), RetVal, UnreachableBlock);
216 ReturningBlocks.push_back(UnreachableBlock);
220 // Now handle return blocks.
221 if (ReturningBlocks.empty())
222 return false; // No blocks return
224 if (ReturningBlocks.size() == 1)
225 return false; // Already has a single return block
227 const TargetTransformInfo &TTI
228 = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
230 unifyReturnBlockSet(F, ReturningBlocks, TTI, "UnifiedReturnBlock");