1 //===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass identifies expensive constants to hoist and coalesces them to
11 // better prepare it for SelectionDAG-based code generation. This works around
12 // the limitations of the basic-block-at-a-time approach.
14 // First it scans all instructions for integer constants and calculates its
15 // cost. If the constant can be folded into the instruction (the cost is
16 // TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't
17 // consider it expensive and leave it alone. This is the default behavior and
18 // the default implementation of getIntImmCost will always return TCC_Free.
20 // If the cost is more than TCC_BASIC, then the integer constant can't be folded
21 // into the instruction and it might be beneficial to hoist the constant.
22 // Similar constants are coalesced to reduce register pressure and
23 // materialization code.
25 // When a constant is hoisted, it is also hidden behind a bitcast to force it to
26 // be live-out of the basic block. Otherwise the constant would be just
27 // duplicated and each basic block would have its own copy in the SelectionDAG.
28 // The SelectionDAG recognizes such constants as opaque and doesn't perform
29 // certain transformations on them, which would create a new expensive constant.
31 // This optimization is only applied to integer constants in instructions and
32 // simple (this means not nested) constant cast expressions. For example:
33 // %0 = load i64* inttoptr (i64 big_constant to i64*)
34 //===----------------------------------------------------------------------===//
36 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
37 #include "llvm/ADT/APInt.h"
38 #include "llvm/ADT/DenseMap.h"
39 #include "llvm/ADT/None.h"
40 #include "llvm/ADT/Optional.h"
41 #include "llvm/ADT/SmallPtrSet.h"
42 #include "llvm/ADT/SmallVector.h"
43 #include "llvm/ADT/Statistic.h"
44 #include "llvm/Analysis/BlockFrequencyInfo.h"
45 #include "llvm/Analysis/TargetTransformInfo.h"
46 #include "llvm/Transforms/Utils/Local.h"
47 #include "llvm/IR/BasicBlock.h"
48 #include "llvm/IR/Constants.h"
49 #include "llvm/IR/DebugInfoMetadata.h"
50 #include "llvm/IR/Dominators.h"
51 #include "llvm/IR/Function.h"
52 #include "llvm/IR/InstrTypes.h"
53 #include "llvm/IR/Instruction.h"
54 #include "llvm/IR/Instructions.h"
55 #include "llvm/IR/IntrinsicInst.h"
56 #include "llvm/IR/Value.h"
57 #include "llvm/Pass.h"
58 #include "llvm/Support/BlockFrequency.h"
59 #include "llvm/Support/Casting.h"
60 #include "llvm/Support/CommandLine.h"
61 #include "llvm/Support/Debug.h"
62 #include "llvm/Support/raw_ostream.h"
63 #include "llvm/Transforms/Scalar.h"
72 using namespace consthoist;
74 #define DEBUG_TYPE "consthoist"
76 STATISTIC(NumConstantsHoisted, "Number of constants hoisted");
77 STATISTIC(NumConstantsRebased, "Number of constants rebased");
79 static cl::opt<bool> ConstHoistWithBlockFrequency(
80 "consthoist-with-block-frequency", cl::init(true), cl::Hidden,
81 cl::desc("Enable the use of the block frequency analysis to reduce the "
82 "chance to execute const materialization more frequently than "
83 "without hoisting."));
85 static cl::opt<bool> ConstHoistGEP(
86 "consthoist-gep", cl::init(false), cl::Hidden,
87 cl::desc("Try hoisting constant gep expressions"));
89 static cl::opt<unsigned>
90 MinNumOfDependentToRebase("consthoist-min-num-to-rebase",
91 cl::desc("Do not rebase if number of dependent constants of a Base is less "
93 cl::init(0), cl::Hidden);
97 /// The constant hoisting pass.
98 class ConstantHoistingLegacyPass : public FunctionPass {
100 static char ID; // Pass identification, replacement for typeid
102 ConstantHoistingLegacyPass() : FunctionPass(ID) {
103 initializeConstantHoistingLegacyPassPass(*PassRegistry::getPassRegistry());
106 bool runOnFunction(Function &Fn) override;
108 StringRef getPassName() const override { return "Constant Hoisting"; }
110 void getAnalysisUsage(AnalysisUsage &AU) const override {
111 AU.setPreservesCFG();
112 if (ConstHoistWithBlockFrequency)
113 AU.addRequired<BlockFrequencyInfoWrapperPass>();
114 AU.addRequired<DominatorTreeWrapperPass>();
115 AU.addRequired<TargetTransformInfoWrapperPass>();
118 void releaseMemory() override { Impl.releaseMemory(); }
121 ConstantHoistingPass Impl;
124 } // end anonymous namespace
126 char ConstantHoistingLegacyPass::ID = 0;
128 INITIALIZE_PASS_BEGIN(ConstantHoistingLegacyPass, "consthoist",
129 "Constant Hoisting", false, false)
130 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
131 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
132 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
133 INITIALIZE_PASS_END(ConstantHoistingLegacyPass, "consthoist",
134 "Constant Hoisting", false, false)
136 FunctionPass *llvm::createConstantHoistingPass() {
137 return new ConstantHoistingLegacyPass();
140 /// Perform the constant hoisting optimization for the given function.
141 bool ConstantHoistingLegacyPass::runOnFunction(Function &Fn) {
142 if (skipFunction(Fn))
145 LLVM_DEBUG(dbgs() << "********** Begin Constant Hoisting **********\n");
146 LLVM_DEBUG(dbgs() << "********** Function: " << Fn.getName() << '\n');
149 Impl.runImpl(Fn, getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn),
150 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
151 ConstHoistWithBlockFrequency
152 ? &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI()
157 LLVM_DEBUG(dbgs() << "********** Function after Constant Hoisting: "
158 << Fn.getName() << '\n');
159 LLVM_DEBUG(dbgs() << Fn);
161 LLVM_DEBUG(dbgs() << "********** End Constant Hoisting **********\n");
166 /// Find the constant materialization insertion point.
167 Instruction *ConstantHoistingPass::findMatInsertPt(Instruction *Inst,
168 unsigned Idx) const {
169 // If the operand is a cast instruction, then we have to materialize the
170 // constant before the cast instruction.
172 Value *Opnd = Inst->getOperand(Idx);
173 if (auto CastInst = dyn_cast<Instruction>(Opnd))
174 if (CastInst->isCast())
178 // The simple and common case. This also includes constant expressions.
179 if (!isa<PHINode>(Inst) && !Inst->isEHPad())
182 // We can't insert directly before a phi node or an eh pad. Insert before
183 // the terminator of the incoming or dominating block.
184 assert(Entry != Inst->getParent() && "PHI or landing pad in entry block!");
185 if (Idx != ~0U && isa<PHINode>(Inst))
186 return cast<PHINode>(Inst)->getIncomingBlock(Idx)->getTerminator();
188 // This must be an EH pad. Iterate over immediate dominators until we find a
189 // non-EH pad. We need to skip over catchswitch blocks, which are both EH pads
191 auto IDom = DT->getNode(Inst->getParent())->getIDom();
192 while (IDom->getBlock()->isEHPad()) {
193 assert(Entry != IDom->getBlock() && "eh pad in entry block");
194 IDom = IDom->getIDom();
197 return IDom->getBlock()->getTerminator();
200 /// Given \p BBs as input, find another set of BBs which collectively
201 /// dominates \p BBs and have the minimal sum of frequencies. Return the BB
202 /// set found in \p BBs.
203 static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI,
205 SmallPtrSet<BasicBlock *, 8> &BBs) {
206 assert(!BBs.count(Entry) && "Assume Entry is not in BBs");
207 // Nodes on the current path to the root.
208 SmallPtrSet<BasicBlock *, 8> Path;
209 // Candidates includes any block 'BB' in set 'BBs' that is not strictly
210 // dominated by any other blocks in set 'BBs', and all nodes in the path
211 // in the dominator tree from Entry to 'BB'.
212 SmallPtrSet<BasicBlock *, 16> Candidates;
213 for (auto BB : BBs) {
215 // Walk up the dominator tree until Entry or another BB in BBs
216 // is reached. Insert the nodes on the way to the Path.
217 BasicBlock *Node = BB;
218 // The "Path" is a candidate path to be added into Candidates set.
219 bool isCandidate = false;
222 if (Node == Entry || Candidates.count(Node)) {
226 assert(DT.getNode(Node)->getIDom() &&
227 "Entry doens't dominate current Node");
228 Node = DT.getNode(Node)->getIDom()->getBlock();
229 } while (!BBs.count(Node));
231 // If isCandidate is false, Node is another Block in BBs dominating
232 // current 'BB'. Drop the nodes on the Path.
236 // Add nodes on the Path into Candidates.
237 Candidates.insert(Path.begin(), Path.end());
240 // Sort the nodes in Candidates in top-down order and save the nodes
243 SmallVector<BasicBlock *, 16> Orders;
244 Orders.push_back(Entry);
245 while (Idx != Orders.size()) {
246 BasicBlock *Node = Orders[Idx++];
247 for (auto ChildDomNode : DT.getNode(Node)->getChildren()) {
248 if (Candidates.count(ChildDomNode->getBlock()))
249 Orders.push_back(ChildDomNode->getBlock());
253 // Visit Orders in bottom-up order.
254 using InsertPtsCostPair =
255 std::pair<SmallPtrSet<BasicBlock *, 16>, BlockFrequency>;
257 // InsertPtsMap is a map from a BB to the best insertion points for the
258 // subtree of BB (subtree not including the BB itself).
259 DenseMap<BasicBlock *, InsertPtsCostPair> InsertPtsMap;
260 InsertPtsMap.reserve(Orders.size() + 1);
261 for (auto RIt = Orders.rbegin(); RIt != Orders.rend(); RIt++) {
262 BasicBlock *Node = *RIt;
263 bool NodeInBBs = BBs.count(Node);
264 SmallPtrSet<BasicBlock *, 16> &InsertPts = InsertPtsMap[Node].first;
265 BlockFrequency &InsertPtsFreq = InsertPtsMap[Node].second;
267 // Return the optimal insert points in BBs.
270 if (InsertPtsFreq > BFI.getBlockFreq(Node) ||
271 (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1))
274 BBs.insert(InsertPts.begin(), InsertPts.end());
278 BasicBlock *Parent = DT.getNode(Node)->getIDom()->getBlock();
279 // Initially, ParentInsertPts is empty and ParentPtsFreq is 0. Every child
280 // will update its parent's ParentInsertPts and ParentPtsFreq.
281 SmallPtrSet<BasicBlock *, 16> &ParentInsertPts = InsertPtsMap[Parent].first;
282 BlockFrequency &ParentPtsFreq = InsertPtsMap[Parent].second;
283 // Choose to insert in Node or in subtree of Node.
284 // Don't hoist to EHPad because we may not find a proper place to insert
286 // If the total frequency of InsertPts is the same as the frequency of the
287 // target Node, and InsertPts contains more than one nodes, choose hoisting
288 // to reduce code size.
291 (InsertPtsFreq > BFI.getBlockFreq(Node) ||
292 (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1)))) {
293 ParentInsertPts.insert(Node);
294 ParentPtsFreq += BFI.getBlockFreq(Node);
296 ParentInsertPts.insert(InsertPts.begin(), InsertPts.end());
297 ParentPtsFreq += InsertPtsFreq;
302 /// Find an insertion point that dominates all uses.
303 SmallPtrSet<Instruction *, 8> ConstantHoistingPass::findConstantInsertionPoint(
304 const ConstantInfo &ConstInfo) const {
305 assert(!ConstInfo.RebasedConstants.empty() && "Invalid constant info entry.");
306 // Collect all basic blocks.
307 SmallPtrSet<BasicBlock *, 8> BBs;
308 SmallPtrSet<Instruction *, 8> InsertPts;
309 for (auto const &RCI : ConstInfo.RebasedConstants)
310 for (auto const &U : RCI.Uses)
311 BBs.insert(findMatInsertPt(U.Inst, U.OpndIdx)->getParent());
313 if (BBs.count(Entry)) {
314 InsertPts.insert(&Entry->front());
319 findBestInsertionSet(*DT, *BFI, Entry, BBs);
320 for (auto BB : BBs) {
321 BasicBlock::iterator InsertPt = BB->begin();
322 for (; isa<PHINode>(InsertPt) || InsertPt->isEHPad(); ++InsertPt)
324 InsertPts.insert(&*InsertPt);
329 while (BBs.size() >= 2) {
330 BasicBlock *BB, *BB1, *BB2;
332 BB2 = *std::next(BBs.begin());
333 BB = DT->findNearestCommonDominator(BB1, BB2);
335 InsertPts.insert(&Entry->front());
342 assert((BBs.size() == 1) && "Expected only one element.");
343 Instruction &FirstInst = (*BBs.begin())->front();
344 InsertPts.insert(findMatInsertPt(&FirstInst));
348 /// Record constant integer ConstInt for instruction Inst at operand
351 /// The operand at index Idx is not necessarily the constant integer itself. It
352 /// could also be a cast instruction or a constant expression that uses the
353 /// constant integer.
354 void ConstantHoistingPass::collectConstantCandidates(
355 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
356 ConstantInt *ConstInt) {
358 // Ask the target about the cost of materializing the constant for the given
359 // instruction and operand index.
360 if (auto IntrInst = dyn_cast<IntrinsicInst>(Inst))
361 Cost = TTI->getIntImmCost(IntrInst->getIntrinsicID(), Idx,
362 ConstInt->getValue(), ConstInt->getType());
364 Cost = TTI->getIntImmCost(Inst->getOpcode(), Idx, ConstInt->getValue(),
365 ConstInt->getType());
367 // Ignore cheap integer constants.
368 if (Cost > TargetTransformInfo::TCC_Basic) {
369 ConstCandMapType::iterator Itr;
371 ConstPtrUnionType Cand = ConstInt;
372 std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
374 ConstIntCandVec.push_back(ConstantCandidate(ConstInt));
375 Itr->second = ConstIntCandVec.size() - 1;
377 ConstIntCandVec[Itr->second].addUser(Inst, Idx, Cost);
378 LLVM_DEBUG(if (isa<ConstantInt>(Inst->getOperand(Idx))) dbgs()
379 << "Collect constant " << *ConstInt << " from " << *Inst
380 << " with cost " << Cost << '\n';
381 else dbgs() << "Collect constant " << *ConstInt
382 << " indirectly from " << *Inst << " via "
383 << *Inst->getOperand(Idx) << " with cost " << Cost
388 /// Record constant GEP expression for instruction Inst at operand index Idx.
389 void ConstantHoistingPass::collectConstantCandidates(
390 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
391 ConstantExpr *ConstExpr) {
392 // TODO: Handle vector GEPs
393 if (ConstExpr->getType()->isVectorTy())
396 GlobalVariable *BaseGV = dyn_cast<GlobalVariable>(ConstExpr->getOperand(0));
400 // Get offset from the base GV.
401 PointerType *GVPtrTy = dyn_cast<PointerType>(BaseGV->getType());
402 IntegerType *PtrIntTy = DL->getIntPtrType(*Ctx, GVPtrTy->getAddressSpace());
403 APInt Offset(DL->getTypeSizeInBits(PtrIntTy), /*val*/0, /*isSigned*/true);
404 auto *GEPO = cast<GEPOperator>(ConstExpr);
405 if (!GEPO->accumulateConstantOffset(*DL, Offset))
408 if (!Offset.isIntN(32))
411 // A constant GEP expression that has a GlobalVariable as base pointer is
412 // usually lowered to a load from constant pool. Such operation is unlikely
413 // to be cheaper than compute it by <Base + Offset>, which can be lowered to
414 // an ADD instruction or folded into Load/Store instruction.
415 int Cost = TTI->getIntImmCost(Instruction::Add, 1, Offset, PtrIntTy);
416 ConstCandVecType &ExprCandVec = ConstGEPCandMap[BaseGV];
417 ConstCandMapType::iterator Itr;
419 ConstPtrUnionType Cand = ConstExpr;
420 std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
422 ExprCandVec.push_back(ConstantCandidate(
423 ConstantInt::get(Type::getInt32Ty(*Ctx), Offset.getLimitedValue()),
425 Itr->second = ExprCandVec.size() - 1;
427 ExprCandVec[Itr->second].addUser(Inst, Idx, Cost);
430 /// Check the operand for instruction Inst at index Idx.
431 void ConstantHoistingPass::collectConstantCandidates(
432 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx) {
433 Value *Opnd = Inst->getOperand(Idx);
435 // Visit constant integers.
436 if (auto ConstInt = dyn_cast<ConstantInt>(Opnd)) {
437 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
441 // Visit cast instructions that have constant integers.
442 if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
443 // Only visit cast instructions, which have been skipped. All other
444 // instructions should have already been visited.
445 if (!CastInst->isCast())
448 if (auto *ConstInt = dyn_cast<ConstantInt>(CastInst->getOperand(0))) {
449 // Pretend the constant is directly used by the instruction and ignore
450 // the cast instruction.
451 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
456 // Visit constant expressions that have constant integers.
457 if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
458 // Handle constant gep expressions.
459 if (ConstHoistGEP && ConstExpr->isGEPWithNoNotionalOverIndexing())
460 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstExpr);
462 // Only visit constant cast expressions.
463 if (!ConstExpr->isCast())
466 if (auto ConstInt = dyn_cast<ConstantInt>(ConstExpr->getOperand(0))) {
467 // Pretend the constant is directly used by the instruction and ignore
468 // the constant expression.
469 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
475 /// Scan the instruction for expensive integer constants and record them
476 /// in the constant candidate vector.
477 void ConstantHoistingPass::collectConstantCandidates(
478 ConstCandMapType &ConstCandMap, Instruction *Inst) {
479 // Skip all cast instructions. They are visited indirectly later on.
483 // Scan all operands.
484 for (unsigned Idx = 0, E = Inst->getNumOperands(); Idx != E; ++Idx) {
485 // The cost of materializing the constants (defined in
486 // `TargetTransformInfo::getIntImmCost`) for instructions which only take
487 // constant variables is lower than `TargetTransformInfo::TCC_Basic`. So
488 // it's safe for us to collect constant candidates from all IntrinsicInsts.
489 if (canReplaceOperandWithVariable(Inst, Idx) || isa<IntrinsicInst>(Inst)) {
490 collectConstantCandidates(ConstCandMap, Inst, Idx);
492 } // end of for all operands
495 /// Collect all integer constants in the function that cannot be folded
496 /// into an instruction itself.
497 void ConstantHoistingPass::collectConstantCandidates(Function &Fn) {
498 ConstCandMapType ConstCandMap;
499 for (BasicBlock &BB : Fn)
500 for (Instruction &Inst : BB)
501 collectConstantCandidates(ConstCandMap, &Inst);
504 // This helper function is necessary to deal with values that have different
505 // bit widths (APInt Operator- does not like that). If the value cannot be
506 // represented in uint64 we return an "empty" APInt. This is then interpreted
507 // as the value is not in range.
508 static Optional<APInt> calculateOffsetDiff(const APInt &V1, const APInt &V2) {
509 Optional<APInt> Res = None;
510 unsigned BW = V1.getBitWidth() > V2.getBitWidth() ?
511 V1.getBitWidth() : V2.getBitWidth();
512 uint64_t LimVal1 = V1.getLimitedValue();
513 uint64_t LimVal2 = V2.getLimitedValue();
515 if (LimVal1 == ~0ULL || LimVal2 == ~0ULL)
518 uint64_t Diff = LimVal1 - LimVal2;
519 return APInt(BW, Diff, true);
522 // From a list of constants, one needs to picked as the base and the other
523 // constants will be transformed into an offset from that base constant. The
524 // question is which we can pick best? For example, consider these constants
525 // and their number of uses:
527 // Constants| 2 | 4 | 12 | 42 |
528 // NumUses | 3 | 2 | 8 | 7 |
530 // Selecting constant 12 because it has the most uses will generate negative
531 // offsets for constants 2 and 4 (i.e. -10 and -8 respectively). If negative
532 // offsets lead to less optimal code generation, then there might be better
533 // solutions. Suppose immediates in the range of 0..35 are most optimally
534 // supported by the architecture, then selecting constant 2 is most optimal
535 // because this will generate offsets: 0, 2, 10, 40. Offsets 0, 2 and 10 are in
536 // range 0..35, and thus 3 + 2 + 8 = 13 uses are in range. Selecting 12 would
537 // have only 8 uses in range, so choosing 2 as a base is more optimal. Thus, in
538 // selecting the base constant the range of the offsets is a very important
539 // factor too that we take into account here. This algorithm calculates a total
540 // costs for selecting a constant as the base and substract the costs if
541 // immediates are out of range. It has quadratic complexity, so we call this
542 // function only when we're optimising for size and there are less than 100
543 // constants, we fall back to the straightforward algorithm otherwise
544 // which does not do all the offset calculations.
546 ConstantHoistingPass::maximizeConstantsInRange(ConstCandVecType::iterator S,
547 ConstCandVecType::iterator E,
548 ConstCandVecType::iterator &MaxCostItr) {
549 unsigned NumUses = 0;
551 if(!Entry->getParent()->optForSize() || std::distance(S,E) > 100) {
552 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
553 NumUses += ConstCand->Uses.size();
554 if (ConstCand->CumulativeCost > MaxCostItr->CumulativeCost)
555 MaxCostItr = ConstCand;
560 LLVM_DEBUG(dbgs() << "== Maximize constants in range ==\n");
562 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
563 auto Value = ConstCand->ConstInt->getValue();
564 Type *Ty = ConstCand->ConstInt->getType();
566 NumUses += ConstCand->Uses.size();
567 LLVM_DEBUG(dbgs() << "= Constant: " << ConstCand->ConstInt->getValue()
570 for (auto User : ConstCand->Uses) {
571 unsigned Opcode = User.Inst->getOpcode();
572 unsigned OpndIdx = User.OpndIdx;
573 Cost += TTI->getIntImmCost(Opcode, OpndIdx, Value, Ty);
574 LLVM_DEBUG(dbgs() << "Cost: " << Cost << "\n");
576 for (auto C2 = S; C2 != E; ++C2) {
577 Optional<APInt> Diff = calculateOffsetDiff(
578 C2->ConstInt->getValue(),
579 ConstCand->ConstInt->getValue());
582 TTI->getIntImmCodeSizeCost(Opcode, OpndIdx, Diff.getValue(), Ty);
584 LLVM_DEBUG(dbgs() << "Offset " << Diff.getValue() << " "
585 << "has penalty: " << ImmCosts << "\n"
586 << "Adjusted cost: " << Cost << "\n");
590 LLVM_DEBUG(dbgs() << "Cumulative cost: " << Cost << "\n");
591 if (Cost > MaxCost) {
593 MaxCostItr = ConstCand;
594 LLVM_DEBUG(dbgs() << "New candidate: " << MaxCostItr->ConstInt->getValue()
601 /// Find the base constant within the given range and rebase all other
602 /// constants with respect to the base constant.
603 void ConstantHoistingPass::findAndMakeBaseConstant(
604 ConstCandVecType::iterator S, ConstCandVecType::iterator E,
605 SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec) {
607 unsigned NumUses = maximizeConstantsInRange(S, E, MaxCostItr);
609 // Don't hoist constants that have only one use.
613 ConstantInt *ConstInt = MaxCostItr->ConstInt;
614 ConstantExpr *ConstExpr = MaxCostItr->ConstExpr;
615 ConstantInfo ConstInfo;
616 ConstInfo.BaseInt = ConstInt;
617 ConstInfo.BaseExpr = ConstExpr;
618 Type *Ty = ConstInt->getType();
620 // Rebase the constants with respect to the base constant.
621 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
622 APInt Diff = ConstCand->ConstInt->getValue() - ConstInt->getValue();
623 Constant *Offset = Diff == 0 ? nullptr : ConstantInt::get(Ty, Diff);
625 ConstCand->ConstExpr ? ConstCand->ConstExpr->getType() : nullptr;
626 ConstInfo.RebasedConstants.push_back(
627 RebasedConstantInfo(std::move(ConstCand->Uses), Offset, ConstTy));
629 ConstInfoVec.push_back(std::move(ConstInfo));
632 /// Finds and combines constant candidates that can be easily
633 /// rematerialized with an add from a common base constant.
634 void ConstantHoistingPass::findBaseConstants(GlobalVariable *BaseGV) {
635 // If BaseGV is nullptr, find base among candidate constant integers;
636 // Otherwise find base among constant GEPs that share the same BaseGV.
637 ConstCandVecType &ConstCandVec = BaseGV ?
638 ConstGEPCandMap[BaseGV] : ConstIntCandVec;
639 ConstInfoVecType &ConstInfoVec = BaseGV ?
640 ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
642 // Sort the constants by value and type. This invalidates the mapping!
643 std::stable_sort(ConstCandVec.begin(), ConstCandVec.end(),
644 [](const ConstantCandidate &LHS, const ConstantCandidate &RHS) {
645 if (LHS.ConstInt->getType() != RHS.ConstInt->getType())
646 return LHS.ConstInt->getType()->getBitWidth() <
647 RHS.ConstInt->getType()->getBitWidth();
648 return LHS.ConstInt->getValue().ult(RHS.ConstInt->getValue());
651 // Simple linear scan through the sorted constant candidate vector for viable
653 auto MinValItr = ConstCandVec.begin();
654 for (auto CC = std::next(ConstCandVec.begin()), E = ConstCandVec.end();
656 if (MinValItr->ConstInt->getType() == CC->ConstInt->getType()) {
657 Type *MemUseValTy = nullptr;
658 for (auto &U : CC->Uses) {
660 if (LoadInst *LI = dyn_cast<LoadInst>(UI)) {
661 MemUseValTy = LI->getType();
663 } else if (StoreInst *SI = dyn_cast<StoreInst>(UI)) {
664 // Make sure the constant is used as pointer operand of the StoreInst.
665 if (SI->getPointerOperand() == SI->getOperand(U.OpndIdx)) {
666 MemUseValTy = SI->getValueOperand()->getType();
672 // Check if the constant is in range of an add with immediate.
673 APInt Diff = CC->ConstInt->getValue() - MinValItr->ConstInt->getValue();
674 if ((Diff.getBitWidth() <= 64) &&
675 TTI->isLegalAddImmediate(Diff.getSExtValue()) &&
676 // Check if Diff can be used as offset in addressing mode of the user
677 // memory instruction.
678 (!MemUseValTy || TTI->isLegalAddressingMode(MemUseValTy,
679 /*BaseGV*/nullptr, /*BaseOffset*/Diff.getSExtValue(),
680 /*HasBaseReg*/true, /*Scale*/0)))
683 // We either have now a different constant type or the constant is not in
684 // range of an add with immediate anymore.
685 findAndMakeBaseConstant(MinValItr, CC, ConstInfoVec);
686 // Start a new base constant search.
689 // Finalize the last base constant search.
690 findAndMakeBaseConstant(MinValItr, ConstCandVec.end(), ConstInfoVec);
693 /// Updates the operand at Idx in instruction Inst with the result of
694 /// instruction Mat. If the instruction is a PHI node then special
695 /// handling for duplicate values form the same incoming basic block is
697 /// \return The update will always succeed, but the return value indicated if
698 /// Mat was used for the update or not.
699 static bool updateOperand(Instruction *Inst, unsigned Idx, Instruction *Mat) {
700 if (auto PHI = dyn_cast<PHINode>(Inst)) {
701 // Check if any previous operand of the PHI node has the same incoming basic
702 // block. This is a very odd case that happens when the incoming basic block
703 // has a switch statement. In this case use the same value as the previous
704 // operand(s), otherwise we will fail verification due to different values.
705 // The values are actually the same, but the variable names are different
706 // and the verifier doesn't like that.
707 BasicBlock *IncomingBB = PHI->getIncomingBlock(Idx);
708 for (unsigned i = 0; i < Idx; ++i) {
709 if (PHI->getIncomingBlock(i) == IncomingBB) {
710 Value *IncomingVal = PHI->getIncomingValue(i);
711 Inst->setOperand(Idx, IncomingVal);
717 Inst->setOperand(Idx, Mat);
721 /// Emit materialization code for all rebased constants and update their
723 void ConstantHoistingPass::emitBaseConstants(Instruction *Base,
726 const ConstantUser &ConstUser) {
727 Instruction *Mat = Base;
729 // The same offset can be dereferenced to different types in nested struct.
730 if (!Offset && Ty && Ty != Base->getType())
731 Offset = ConstantInt::get(Type::getInt32Ty(*Ctx), 0);
734 Instruction *InsertionPt = findMatInsertPt(ConstUser.Inst,
737 // Constant being rebased is a ConstantExpr.
738 PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx,
739 cast<PointerType>(Ty)->getAddressSpace());
740 Base = new BitCastInst(Base, Int8PtrTy, "base_bitcast", InsertionPt);
741 Mat = GetElementPtrInst::Create(Int8PtrTy->getElementType(), Base,
742 Offset, "mat_gep", InsertionPt);
743 Mat = new BitCastInst(Mat, Ty, "mat_bitcast", InsertionPt);
745 // Constant being rebased is a ConstantInt.
746 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
747 "const_mat", InsertionPt);
749 LLVM_DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)
750 << " + " << *Offset << ") in BB "
751 << Mat->getParent()->getName() << '\n'
753 Mat->setDebugLoc(ConstUser.Inst->getDebugLoc());
755 Value *Opnd = ConstUser.Inst->getOperand(ConstUser.OpndIdx);
757 // Visit constant integer.
758 if (isa<ConstantInt>(Opnd)) {
759 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
760 if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat) && Offset)
761 Mat->eraseFromParent();
762 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n');
766 // Visit cast instruction.
767 if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
768 assert(CastInst->isCast() && "Expected an cast instruction!");
769 // Check if we already have visited this cast instruction before to avoid
770 // unnecessary cloning.
771 Instruction *&ClonedCastInst = ClonedCastMap[CastInst];
772 if (!ClonedCastInst) {
773 ClonedCastInst = CastInst->clone();
774 ClonedCastInst->setOperand(0, Mat);
775 ClonedCastInst->insertAfter(CastInst);
776 // Use the same debug location as the original cast instruction.
777 ClonedCastInst->setDebugLoc(CastInst->getDebugLoc());
778 LLVM_DEBUG(dbgs() << "Clone instruction: " << *CastInst << '\n'
779 << "To : " << *ClonedCastInst << '\n');
782 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
783 updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ClonedCastInst);
784 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n');
788 // Visit constant expression.
789 if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
790 if (ConstExpr->isGEPWithNoNotionalOverIndexing()) {
791 // Operand is a ConstantGEP, replace it.
792 updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat);
796 // Aside from constant GEPs, only constant cast expressions are collected.
797 assert(ConstExpr->isCast() && "ConstExpr should be a cast");
798 Instruction *ConstExprInst = ConstExpr->getAsInstruction();
799 ConstExprInst->setOperand(0, Mat);
800 ConstExprInst->insertBefore(findMatInsertPt(ConstUser.Inst,
803 // Use the same debug location as the instruction we are about to update.
804 ConstExprInst->setDebugLoc(ConstUser.Inst->getDebugLoc());
806 LLVM_DEBUG(dbgs() << "Create instruction: " << *ConstExprInst << '\n'
807 << "From : " << *ConstExpr << '\n');
808 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n');
809 if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ConstExprInst)) {
810 ConstExprInst->eraseFromParent();
812 Mat->eraseFromParent();
814 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n');
819 /// Hoist and hide the base constant behind a bitcast and emit
820 /// materialization code for derived constants.
821 bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) {
822 bool MadeChange = false;
823 SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec =
824 BaseGV ? ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
825 for (auto const &ConstInfo : ConstInfoVec) {
826 SmallPtrSet<Instruction *, 8> IPSet = findConstantInsertionPoint(ConstInfo);
827 assert(!IPSet.empty() && "IPSet is empty");
829 unsigned UsesNum = 0;
830 unsigned ReBasesNum = 0;
831 unsigned NotRebasedNum = 0;
832 for (Instruction *IP : IPSet) {
833 // First, collect constants depending on this IP of the base.
835 using RebasedUse = std::tuple<Constant *, Type *, ConstantUser>;
836 SmallVector<RebasedUse, 4> ToBeRebased;
837 for (auto const &RCI : ConstInfo.RebasedConstants) {
838 for (auto const &U : RCI.Uses) {
840 BasicBlock *OrigMatInsertBB =
841 findMatInsertPt(U.Inst, U.OpndIdx)->getParent();
842 // If Base constant is to be inserted in multiple places,
843 // generate rebase for U using the Base dominating U.
844 if (IPSet.size() == 1 ||
845 DT->dominates(IP->getParent(), OrigMatInsertBB))
846 ToBeRebased.push_back(RebasedUse(RCI.Offset, RCI.Ty, U));
851 // If only few constants depend on this IP of base, skip rebasing,
852 // assuming the base and the rebased have the same materialization cost.
853 if (ToBeRebased.size() < MinNumOfDependentToRebase) {
854 NotRebasedNum += ToBeRebased.size();
858 // Emit an instance of the base at this IP.
859 Instruction *Base = nullptr;
860 // Hoist and hide the base constant behind a bitcast.
861 if (ConstInfo.BaseExpr) {
862 assert(BaseGV && "A base constant expression must have an base GV");
863 Type *Ty = ConstInfo.BaseExpr->getType();
864 Base = new BitCastInst(ConstInfo.BaseExpr, Ty, "const", IP);
866 IntegerType *Ty = ConstInfo.BaseInt->getType();
867 Base = new BitCastInst(ConstInfo.BaseInt, Ty, "const", IP);
870 Base->setDebugLoc(IP->getDebugLoc());
872 LLVM_DEBUG(dbgs() << "Hoist constant (" << *ConstInfo.BaseInt
873 << ") to BB " << IP->getParent()->getName() << '\n'
876 // Emit materialization code for rebased constants depending on this IP.
877 for (auto const &R : ToBeRebased) {
878 Constant *Off = std::get<0>(R);
879 Type *Ty = std::get<1>(R);
880 ConstantUser U = std::get<2>(R);
881 emitBaseConstants(Base, Off, Ty, U);
883 // Use the same debug location as the last user of the constant.
884 Base->setDebugLoc(DILocation::getMergedLocation(
885 Base->getDebugLoc(), U.Inst->getDebugLoc()));
887 assert(!Base->use_empty() && "The use list is empty!?");
888 assert(isa<Instruction>(Base->user_back()) &&
889 "All uses should be instructions.");
894 // Expect all uses are rebased after rebase is done.
895 assert(UsesNum == (ReBasesNum + NotRebasedNum) &&
896 "Not all uses are rebased");
898 NumConstantsHoisted++;
900 // Base constant is also included in ConstInfo.RebasedConstants, so
901 // deduct 1 from ConstInfo.RebasedConstants.size().
902 NumConstantsRebased += ConstInfo.RebasedConstants.size() - 1;
909 /// Check all cast instructions we made a copy of and remove them if they
910 /// have no more users.
911 void ConstantHoistingPass::deleteDeadCastInst() const {
912 for (auto const &I : ClonedCastMap)
913 if (I.first->use_empty())
914 I.first->eraseFromParent();
917 /// Optimize expensive integer constants in the given function.
918 bool ConstantHoistingPass::runImpl(Function &Fn, TargetTransformInfo &TTI,
919 DominatorTree &DT, BlockFrequencyInfo *BFI,
924 this->DL = &Fn.getParent()->getDataLayout();
925 this->Ctx = &Fn.getContext();
926 this->Entry = &Entry;
927 // Collect all constant candidates.
928 collectConstantCandidates(Fn);
930 // Combine constants that can be easily materialized with an add from a common
932 if (!ConstIntCandVec.empty())
933 findBaseConstants(nullptr);
934 for (auto &MapEntry : ConstGEPCandMap)
935 if (!MapEntry.second.empty())
936 findBaseConstants(MapEntry.first);
938 // Finally hoist the base constant and emit materialization code for dependent
940 bool MadeChange = false;
941 if (!ConstIntInfoVec.empty())
942 MadeChange = emitBaseConstants(nullptr);
943 for (auto MapEntry : ConstGEPInfoMap)
944 if (!MapEntry.second.empty())
945 MadeChange |= emitBaseConstants(MapEntry.first);
948 // Cleanup dead instructions.
949 deleteDeadCastInst();
954 PreservedAnalyses ConstantHoistingPass::run(Function &F,
955 FunctionAnalysisManager &AM) {
956 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
957 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
958 auto BFI = ConstHoistWithBlockFrequency
959 ? &AM.getResult<BlockFrequencyAnalysis>(F)
961 if (!runImpl(F, TTI, DT, BFI, F.getEntryBlock()))
962 return PreservedAnalyses::all();
964 PreservedAnalyses PA;
965 PA.preserveSet<CFGAnalyses>();