1 //===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass hoists instructions to enable speculative execution on
11 // targets where branches are expensive. This is aimed at GPUs. It
12 // currently works on simple if-then and if-then-else
15 // Removing branches is not the only motivation for this
16 // pass. E.g. consider this code and assume that there is no
17 // addressing mode for multiplying by sizeof(*a):
33 // which could later be optimized to
41 // Later passes sink back much of the speculated code that did not enable
42 // further optimization.
44 // This pass is more aggressive than the function SpeculativeyExecuteBB in
45 // SimplifyCFG. SimplifyCFG will not speculate if no selects are introduced and
46 // it will speculate at most one instruction. It also will not speculate if
47 // there is a value defined in the if-block that is only used in the then-block.
48 // These restrictions make sense since the speculation in SimplifyCFG seems
49 // aimed at introducing cheap selects, while this pass is intended to do more
50 // aggressive speculation while counting on later passes to either capitalize on
51 // that or clean it up.
53 // If the pass was created by calling
54 // createSpeculativeExecutionIfHasBranchDivergencePass or the
55 // -spec-exec-only-if-divergent-target option is present, this pass only has an
56 // effect on targets where TargetTransformInfo::hasBranchDivergence() is true;
57 // on other targets, it is a nop.
59 // This lets you include this pass unconditionally in the IR pass pipeline, but
60 // only enable it for relevant targets.
62 //===----------------------------------------------------------------------===//
64 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
65 #include "llvm/ADT/SmallSet.h"
66 #include "llvm/Analysis/GlobalsModRef.h"
67 #include "llvm/Analysis/ValueTracking.h"
68 #include "llvm/IR/Instructions.h"
69 #include "llvm/IR/Module.h"
70 #include "llvm/IR/Operator.h"
71 #include "llvm/Support/CommandLine.h"
72 #include "llvm/Support/Debug.h"
76 #define DEBUG_TYPE "speculative-execution"
78 // The risk that speculation will not pay off increases with the
79 // number of instructions speculated, so we put a limit on that.
80 static cl::opt<unsigned> SpecExecMaxSpeculationCost(
81 "spec-exec-max-speculation-cost", cl::init(7), cl::Hidden,
82 cl::desc("Speculative execution is not applied to basic blocks where "
83 "the cost of the instructions to speculatively execute "
84 "exceeds this limit."));
86 // Speculating just a few instructions from a larger block tends not
87 // to be profitable and this limit prevents that. A reason for that is
88 // that small basic blocks are more likely to be candidates for
89 // further optimization.
90 static cl::opt<unsigned> SpecExecMaxNotHoisted(
91 "spec-exec-max-not-hoisted", cl::init(5), cl::Hidden,
92 cl::desc("Speculative execution is not applied to basic blocks where the "
93 "number of instructions that would not be speculatively executed "
94 "exceeds this limit."));
96 static cl::opt<bool> SpecExecOnlyIfDivergentTarget(
97 "spec-exec-only-if-divergent-target", cl::init(false), cl::Hidden,
98 cl::desc("Speculative execution is applied only to targets with divergent "
99 "branches, even if the pass was configured to apply only to all "
104 class SpeculativeExecutionLegacyPass : public FunctionPass {
107 explicit SpeculativeExecutionLegacyPass(bool OnlyIfDivergentTarget = false)
108 : FunctionPass(ID), OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
109 SpecExecOnlyIfDivergentTarget),
110 Impl(OnlyIfDivergentTarget) {}
112 void getAnalysisUsage(AnalysisUsage &AU) const override;
113 bool runOnFunction(Function &F) override;
115 StringRef getPassName() const override {
116 if (OnlyIfDivergentTarget)
117 return "Speculatively execute instructions if target has divergent "
119 return "Speculatively execute instructions";
123 // Variable preserved purely for correct name printing.
124 const bool OnlyIfDivergentTarget;
126 SpeculativeExecutionPass Impl;
130 char SpeculativeExecutionLegacyPass::ID = 0;
131 INITIALIZE_PASS_BEGIN(SpeculativeExecutionLegacyPass, "speculative-execution",
132 "Speculatively execute instructions", false, false)
133 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
134 INITIALIZE_PASS_END(SpeculativeExecutionLegacyPass, "speculative-execution",
135 "Speculatively execute instructions", false, false)
137 void SpeculativeExecutionLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
138 AU.addRequired<TargetTransformInfoWrapperPass>();
139 AU.addPreserved<GlobalsAAWrapperPass>();
142 bool SpeculativeExecutionLegacyPass::runOnFunction(Function &F) {
146 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
147 return Impl.runImpl(F, TTI);
152 bool SpeculativeExecutionPass::runImpl(Function &F, TargetTransformInfo *TTI) {
153 if (OnlyIfDivergentTarget && !TTI->hasBranchDivergence()) {
154 DEBUG(dbgs() << "Not running SpeculativeExecution because "
155 "TTI->hasBranchDivergence() is false.\n");
160 bool Changed = false;
162 Changed |= runOnBasicBlock(B);
167 bool SpeculativeExecutionPass::runOnBasicBlock(BasicBlock &B) {
168 BranchInst *BI = dyn_cast<BranchInst>(B.getTerminator());
172 if (BI->getNumSuccessors() != 2)
174 BasicBlock &Succ0 = *BI->getSuccessor(0);
175 BasicBlock &Succ1 = *BI->getSuccessor(1);
177 if (&B == &Succ0 || &B == &Succ1 || &Succ0 == &Succ1) {
181 // Hoist from if-then (triangle).
182 if (Succ0.getSinglePredecessor() != nullptr &&
183 Succ0.getSingleSuccessor() == &Succ1) {
184 return considerHoistingFromTo(Succ0, B);
187 // Hoist from if-else (triangle).
188 if (Succ1.getSinglePredecessor() != nullptr &&
189 Succ1.getSingleSuccessor() == &Succ0) {
190 return considerHoistingFromTo(Succ1, B);
193 // Hoist from if-then-else (diamond), but only if it is equivalent to
194 // an if-else or if-then due to one of the branches doing nothing.
195 if (Succ0.getSinglePredecessor() != nullptr &&
196 Succ1.getSinglePredecessor() != nullptr &&
197 Succ1.getSingleSuccessor() != nullptr &&
198 Succ1.getSingleSuccessor() != &B &&
199 Succ1.getSingleSuccessor() == Succ0.getSingleSuccessor()) {
200 // If a block has only one instruction, then that is a terminator
201 // instruction so that the block does nothing. This does happen.
202 if (Succ1.size() == 1) // equivalent to if-then
203 return considerHoistingFromTo(Succ0, B);
204 if (Succ0.size() == 1) // equivalent to if-else
205 return considerHoistingFromTo(Succ1, B);
211 static unsigned ComputeSpeculationCost(const Instruction *I,
212 const TargetTransformInfo &TTI) {
213 switch (Operator::getOpcode(I)) {
214 case Instruction::GetElementPtr:
215 case Instruction::Add:
216 case Instruction::Mul:
217 case Instruction::And:
218 case Instruction::Or:
219 case Instruction::Select:
220 case Instruction::Shl:
221 case Instruction::Sub:
222 case Instruction::LShr:
223 case Instruction::AShr:
224 case Instruction::Xor:
225 case Instruction::ZExt:
226 case Instruction::SExt:
227 case Instruction::Call:
228 case Instruction::BitCast:
229 case Instruction::PtrToInt:
230 case Instruction::IntToPtr:
231 case Instruction::AddrSpaceCast:
232 case Instruction::FPToUI:
233 case Instruction::FPToSI:
234 case Instruction::UIToFP:
235 case Instruction::SIToFP:
236 case Instruction::FPExt:
237 case Instruction::FPTrunc:
238 case Instruction::FAdd:
239 case Instruction::FSub:
240 case Instruction::FMul:
241 case Instruction::FDiv:
242 case Instruction::FRem:
243 case Instruction::ICmp:
244 case Instruction::FCmp:
245 return TTI.getUserCost(I);
248 return UINT_MAX; // Disallow anything not whitelisted.
252 bool SpeculativeExecutionPass::considerHoistingFromTo(
253 BasicBlock &FromBlock, BasicBlock &ToBlock) {
254 SmallSet<const Instruction *, 8> NotHoisted;
255 const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](User *U) {
256 for (Value* V : U->operand_values()) {
257 if (Instruction *I = dyn_cast<Instruction>(V)) {
258 if (NotHoisted.count(I) > 0)
265 unsigned TotalSpeculationCost = 0;
266 for (auto& I : FromBlock) {
267 const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
268 if (Cost != UINT_MAX && isSafeToSpeculativelyExecute(&I) &&
269 AllPrecedingUsesFromBlockHoisted(&I)) {
270 TotalSpeculationCost += Cost;
271 if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
272 return false; // too much to hoist
274 NotHoisted.insert(&I);
275 if (NotHoisted.size() > SpecExecMaxNotHoisted)
276 return false; // too much left behind
280 if (TotalSpeculationCost == 0)
281 return false; // nothing to hoist
283 for (auto I = FromBlock.begin(); I != FromBlock.end();) {
284 // We have to increment I before moving Current as moving Current
285 // changes the list that I is iterating through.
288 if (!NotHoisted.count(&*Current)) {
289 Current->moveBefore(ToBlock.getTerminator());
295 FunctionPass *createSpeculativeExecutionPass() {
296 return new SpeculativeExecutionLegacyPass();
299 FunctionPass *createSpeculativeExecutionIfHasBranchDivergencePass() {
300 return new SpeculativeExecutionLegacyPass(/* OnlyIfDivergentTarget = */ true);
303 SpeculativeExecutionPass::SpeculativeExecutionPass(bool OnlyIfDivergentTarget)
304 : OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
305 SpecExecOnlyIfDivergentTarget) {}
307 PreservedAnalyses SpeculativeExecutionPass::run(Function &F,
308 FunctionAnalysisManager &AM) {
309 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
311 bool Changed = runImpl(F, TTI);
314 return PreservedAnalyses::all();
315 PreservedAnalyses PA;
316 PA.preserve<GlobalsAA>();