1 //===- StructurizeCFG.cpp -------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "llvm/ADT/DenseMap.h"
10 #include "llvm/ADT/MapVector.h"
11 #include "llvm/ADT/PostOrderIterator.h"
12 #include "llvm/ADT/STLExtras.h"
13 #include "llvm/ADT/SmallPtrSet.h"
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/Analysis/InstructionSimplify.h"
16 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
17 #include "llvm/Analysis/LoopInfo.h"
18 #include "llvm/Analysis/RegionInfo.h"
19 #include "llvm/Analysis/RegionIterator.h"
20 #include "llvm/Analysis/RegionPass.h"
21 #include "llvm/IR/Argument.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/CFG.h"
24 #include "llvm/IR/Constant.h"
25 #include "llvm/IR/Constants.h"
26 #include "llvm/IR/Dominators.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/InstrTypes.h"
29 #include "llvm/IR/Instruction.h"
30 #include "llvm/IR/Instructions.h"
31 #include "llvm/IR/Metadata.h"
32 #include "llvm/IR/PatternMatch.h"
33 #include "llvm/IR/Type.h"
34 #include "llvm/IR/Use.h"
35 #include "llvm/IR/User.h"
36 #include "llvm/IR/Value.h"
37 #include "llvm/InitializePasses.h"
38 #include "llvm/Pass.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/CommandLine.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/raw_ostream.h"
44 #include "llvm/Transforms/Scalar.h"
45 #include "llvm/Transforms/Utils.h"
46 #include "llvm/Transforms/Utils/SSAUpdater.h"
52 using namespace llvm::PatternMatch;
54 #define DEBUG_TYPE "structurizecfg"
56 // The name for newly created blocks.
57 static const char *const FlowBlockName = "Flow";
61 static cl::opt<bool> ForceSkipUniformRegions(
62 "structurizecfg-skip-uniform-regions",
64 cl::desc("Force whether the StructurizeCFG pass skips uniform regions"),
68 RelaxedUniformRegions("structurizecfg-relaxed-uniform-regions", cl::Hidden,
69 cl::desc("Allow relaxed uniform region checks"),
72 // Definition of the complex types used in this pass.
74 using BBValuePair = std::pair<BasicBlock *, Value *>;
76 using RNVector = SmallVector<RegionNode *, 8>;
77 using BBVector = SmallVector<BasicBlock *, 8>;
78 using BranchVector = SmallVector<BranchInst *, 8>;
79 using BBValueVector = SmallVector<BBValuePair, 2>;
81 using BBSet = SmallPtrSet<BasicBlock *, 8>;
83 using PhiMap = MapVector<PHINode *, BBValueVector>;
84 using BB2BBVecMap = MapVector<BasicBlock *, BBVector>;
86 using BBPhiMap = DenseMap<BasicBlock *, PhiMap>;
87 using BBPredicates = DenseMap<BasicBlock *, Value *>;
88 using PredMap = DenseMap<BasicBlock *, BBPredicates>;
89 using BB2BBMap = DenseMap<BasicBlock *, BasicBlock *>;
91 /// Finds the nearest common dominator of a set of BasicBlocks.
93 /// For every BB you add to the set, you can specify whether we "remember" the
94 /// block. When you get the common dominator, you can also ask whether it's one
95 /// of the blocks we remembered.
96 class NearestCommonDominator {
98 BasicBlock *Result = nullptr;
99 bool ResultIsRemembered = false;
101 /// Add BB to the resulting dominator.
102 void addBlock(BasicBlock *BB, bool Remember) {
105 ResultIsRemembered = Remember;
109 BasicBlock *NewResult = DT->findNearestCommonDominator(Result, BB);
110 if (NewResult != Result)
111 ResultIsRemembered = false;
113 ResultIsRemembered |= Remember;
118 explicit NearestCommonDominator(DominatorTree *DomTree) : DT(DomTree) {}
120 void addBlock(BasicBlock *BB) {
121 addBlock(BB, /* Remember = */ false);
124 void addAndRememberBlock(BasicBlock *BB) {
125 addBlock(BB, /* Remember = */ true);
128 /// Get the nearest common dominator of all the BBs added via addBlock() and
129 /// addAndRememberBlock().
130 BasicBlock *result() { return Result; }
132 /// Is the BB returned by getResult() one of the blocks we added to the set
133 /// with addAndRememberBlock()?
134 bool resultIsRememberedBlock() { return ResultIsRemembered; }
137 /// Transforms the control flow graph on one single entry/exit region
140 /// After the transform all "If"/"Then"/"Else" style control flow looks like
152 /// | | 1 = "If" block, calculates the condition
153 /// 4 | 2 = "Then" subregion, runs if the condition is true
154 /// | / 3 = "Flow" blocks, newly inserted flow blocks, rejoins the flow
155 /// |/ 4 = "Else" optional subregion, runs if the condition is false
156 /// 5 5 = "End" block, also rejoins the control flow
159 /// Control flow is expressed as a branch where the true exit goes into the
160 /// "Then"/"Else" region, while the false exit skips the region
161 /// The condition for the optional "Else" region is expressed as a PHI node.
162 /// The incoming values of the PHI node are true for the "If" edge and false
163 /// for the "Then" edge.
165 /// Additionally to that even complicated loops look like this:
172 /// | / 1 = "Entry" block
173 /// |/ 2 = "Loop" optional subregion, with all exits at "Flow" block
174 /// 3 3 = "Flow" block, with back edge to entry block
178 /// The back edge of the "Flow" block is always on the false side of the branch
179 /// while the true side continues the general flow. So the loop condition
180 /// consist of a network of PHI nodes where the true incoming values expresses
181 /// breaks and the false values expresses continue states.
182 class StructurizeCFG : public RegionPass {
183 bool SkipUniformRegions;
186 ConstantInt *BoolTrue;
187 ConstantInt *BoolFalse;
188 UndefValue *BoolUndef;
191 Region *ParentRegion;
193 LegacyDivergenceAnalysis *DA;
197 SmallVector<RegionNode *, 8> Order;
200 BBPhiMap DeletedPhis;
201 BB2BBVecMap AddedPhis;
204 BranchVector Conditions;
208 BranchVector LoopConds;
210 RegionNode *PrevNode;
214 Loop *getAdjustedLoop(RegionNode *RN);
215 unsigned getAdjustedLoopDepth(RegionNode *RN);
217 void analyzeLoops(RegionNode *N);
219 Value *invert(Value *Condition);
221 Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
223 void gatherPredicates(RegionNode *N);
227 void insertConditions(bool Loops);
229 void delPhiValues(BasicBlock *From, BasicBlock *To);
231 void addPhiValues(BasicBlock *From, BasicBlock *To);
235 void killTerminator(BasicBlock *BB);
237 void changeExit(RegionNode *Node, BasicBlock *NewExit,
238 bool IncludeDominator);
240 BasicBlock *getNextFlow(BasicBlock *Dominator);
242 BasicBlock *needPrefix(bool NeedEmpty);
244 BasicBlock *needPostfix(BasicBlock *Flow, bool ExitUseAllowed);
246 void setPrevNode(BasicBlock *BB);
248 bool dominatesPredicates(BasicBlock *BB, RegionNode *Node);
250 bool isPredictableTrue(RegionNode *Node);
252 void wireFlow(bool ExitUseAllowed, BasicBlock *LoopEnd);
254 void handleLoops(bool ExitUseAllowed, BasicBlock *LoopEnd);
263 explicit StructurizeCFG(bool SkipUniformRegions_ = false)
265 SkipUniformRegions(SkipUniformRegions_) {
266 if (ForceSkipUniformRegions.getNumOccurrences())
267 SkipUniformRegions = ForceSkipUniformRegions.getValue();
268 initializeStructurizeCFGPass(*PassRegistry::getPassRegistry());
271 bool doInitialization(Region *R, RGPassManager &RGM) override;
273 bool runOnRegion(Region *R, RGPassManager &RGM) override;
275 StringRef getPassName() const override { return "Structurize control flow"; }
277 void getAnalysisUsage(AnalysisUsage &AU) const override {
278 if (SkipUniformRegions)
279 AU.addRequired<LegacyDivergenceAnalysis>();
280 AU.addRequiredID(LowerSwitchID);
281 AU.addRequired<DominatorTreeWrapperPass>();
282 AU.addRequired<LoopInfoWrapperPass>();
284 AU.addPreserved<DominatorTreeWrapperPass>();
285 RegionPass::getAnalysisUsage(AU);
289 } // end anonymous namespace
291 char StructurizeCFG::ID = 0;
293 INITIALIZE_PASS_BEGIN(StructurizeCFG, "structurizecfg", "Structurize the CFG",
295 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
296 INITIALIZE_PASS_DEPENDENCY(LowerSwitch)
297 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
298 INITIALIZE_PASS_DEPENDENCY(RegionInfoPass)
299 INITIALIZE_PASS_END(StructurizeCFG, "structurizecfg", "Structurize the CFG",
302 /// Initialize the types and constants used in the pass
303 bool StructurizeCFG::doInitialization(Region *R, RGPassManager &RGM) {
304 LLVMContext &Context = R->getEntry()->getContext();
306 Boolean = Type::getInt1Ty(Context);
307 BoolTrue = ConstantInt::getTrue(Context);
308 BoolFalse = ConstantInt::getFalse(Context);
309 BoolUndef = UndefValue::get(Boolean);
314 /// Use the exit block to determine the loop if RN is a SubRegion.
315 Loop *StructurizeCFG::getAdjustedLoop(RegionNode *RN) {
316 if (RN->isSubRegion()) {
317 Region *SubRegion = RN->getNodeAs<Region>();
318 return LI->getLoopFor(SubRegion->getExit());
321 return LI->getLoopFor(RN->getEntry());
324 /// Use the exit block to determine the loop depth if RN is a SubRegion.
325 unsigned StructurizeCFG::getAdjustedLoopDepth(RegionNode *RN) {
326 if (RN->isSubRegion()) {
327 Region *SubR = RN->getNodeAs<Region>();
328 return LI->getLoopDepth(SubR->getExit());
331 return LI->getLoopDepth(RN->getEntry());
334 /// Build up the general order of nodes
335 void StructurizeCFG::orderNodes() {
336 ReversePostOrderTraversal<Region*> RPOT(ParentRegion);
337 SmallDenseMap<Loop*, unsigned, 8> LoopBlocks;
339 // The reverse post-order traversal of the list gives us an ordering close
340 // to what we want. The only problem with it is that sometimes backedges
341 // for outer loops will be visited before backedges for inner loops.
342 for (RegionNode *RN : RPOT) {
343 Loop *Loop = getAdjustedLoop(RN);
347 unsigned CurrentLoopDepth = 0;
348 Loop *CurrentLoop = nullptr;
349 for (auto I = RPOT.begin(), E = RPOT.end(); I != E; ++I) {
350 RegionNode *RN = cast<RegionNode>(*I);
351 unsigned LoopDepth = getAdjustedLoopDepth(RN);
353 if (is_contained(Order, *I))
356 if (LoopDepth < CurrentLoopDepth) {
357 // Make sure we have visited all blocks in this loop before moving back to
361 while (unsigned &BlockCount = LoopBlocks[CurrentLoop]) {
363 if (getAdjustedLoop(cast<RegionNode>(*LoopI)) == CurrentLoop) {
365 Order.push_back(*LoopI);
370 CurrentLoop = getAdjustedLoop(RN);
372 LoopBlocks[CurrentLoop]--;
374 CurrentLoopDepth = LoopDepth;
378 // This pass originally used a post-order traversal and then operated on
379 // the list in reverse. Now that we are using a reverse post-order traversal
380 // rather than re-working the whole pass to operate on the list in order,
381 // we just reverse the list and continue to operate on it in reverse.
382 std::reverse(Order.begin(), Order.end());
385 /// Determine the end of the loops
386 void StructurizeCFG::analyzeLoops(RegionNode *N) {
387 if (N->isSubRegion()) {
388 // Test for exit as back edge
389 BasicBlock *Exit = N->getNodeAs<Region>()->getExit();
390 if (Visited.count(Exit))
391 Loops[Exit] = N->getEntry();
394 // Test for successors as back edge
395 BasicBlock *BB = N->getNodeAs<BasicBlock>();
396 BranchInst *Term = cast<BranchInst>(BB->getTerminator());
398 for (BasicBlock *Succ : Term->successors())
399 if (Visited.count(Succ))
404 /// Invert the given condition
405 Value *StructurizeCFG::invert(Value *Condition) {
406 // First: Check if it's a constant
407 if (Constant *C = dyn_cast<Constant>(Condition))
408 return ConstantExpr::getNot(C);
410 // Second: If the condition is already inverted, return the original value
412 if (match(Condition, m_Not(m_Value(NotCondition))))
415 if (Instruction *Inst = dyn_cast<Instruction>(Condition)) {
416 // Third: Check all the users for an invert
417 BasicBlock *Parent = Inst->getParent();
418 for (User *U : Condition->users())
419 if (Instruction *I = dyn_cast<Instruction>(U))
420 if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
423 // Last option: Create a new instruction
424 return BinaryOperator::CreateNot(Condition, "", Parent->getTerminator());
427 if (Argument *Arg = dyn_cast<Argument>(Condition)) {
428 BasicBlock &EntryBlock = Arg->getParent()->getEntryBlock();
429 return BinaryOperator::CreateNot(Condition,
430 Arg->getName() + ".inv",
431 EntryBlock.getTerminator());
434 llvm_unreachable("Unhandled condition to invert");
437 /// Build the condition for one edge
438 Value *StructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
440 Value *Cond = Invert ? BoolFalse : BoolTrue;
441 if (Term->isConditional()) {
442 Cond = Term->getCondition();
444 if (Idx != (unsigned)Invert)
450 /// Analyze the predecessors of each block and build up predicates
451 void StructurizeCFG::gatherPredicates(RegionNode *N) {
452 RegionInfo *RI = ParentRegion->getRegionInfo();
453 BasicBlock *BB = N->getEntry();
454 BBPredicates &Pred = Predicates[BB];
455 BBPredicates &LPred = LoopPreds[BB];
457 for (BasicBlock *P : predecessors(BB)) {
458 // Ignore it if it's a branch from outside into our region entry
459 if (!ParentRegion->contains(P))
462 Region *R = RI->getRegionFor(P);
463 if (R == ParentRegion) {
464 // It's a top level block in our region
465 BranchInst *Term = cast<BranchInst>(P->getTerminator());
466 for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
467 BasicBlock *Succ = Term->getSuccessor(i);
471 if (Visited.count(P)) {
472 // Normal forward edge
473 if (Term->isConditional()) {
474 // Try to treat it like an ELSE block
475 BasicBlock *Other = Term->getSuccessor(!i);
476 if (Visited.count(Other) && !Loops.count(Other) &&
477 !Pred.count(Other) && !Pred.count(P)) {
479 Pred[Other] = BoolFalse;
484 Pred[P] = buildCondition(Term, i, false);
487 LPred[P] = buildCondition(Term, i, true);
491 // It's an exit from a sub region
492 while (R->getParent() != ParentRegion)
495 // Edge from inside a subregion to its entry, ignore it
499 BasicBlock *Entry = R->getEntry();
500 if (Visited.count(Entry))
501 Pred[Entry] = BoolTrue;
503 LPred[Entry] = BoolFalse;
508 /// Collect various loop and predicate infos
509 void StructurizeCFG::collectInfos() {
517 // Reset the visited nodes
520 for (RegionNode *RN : reverse(Order)) {
521 LLVM_DEBUG(dbgs() << "Visiting: "
522 << (RN->isSubRegion() ? "SubRegion with entry: " : "")
523 << RN->getEntry()->getName() << " Loop Depth: "
524 << LI->getLoopDepth(RN->getEntry()) << "\n");
526 // Analyze all the conditions leading to a node
527 gatherPredicates(RN);
529 // Remember that we've seen this node
530 Visited.insert(RN->getEntry());
532 // Find the last back edges
537 /// Insert the missing branch conditions
538 void StructurizeCFG::insertConditions(bool Loops) {
539 BranchVector &Conds = Loops ? LoopConds : Conditions;
540 Value *Default = Loops ? BoolTrue : BoolFalse;
541 SSAUpdater PhiInserter;
543 for (BranchInst *Term : Conds) {
544 assert(Term->isConditional());
546 BasicBlock *Parent = Term->getParent();
547 BasicBlock *SuccTrue = Term->getSuccessor(0);
548 BasicBlock *SuccFalse = Term->getSuccessor(1);
550 PhiInserter.Initialize(Boolean, "");
551 PhiInserter.AddAvailableValue(&Func->getEntryBlock(), Default);
552 PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
554 BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
556 NearestCommonDominator Dominator(DT);
557 Dominator.addBlock(Parent);
559 Value *ParentValue = nullptr;
560 for (std::pair<BasicBlock *, Value *> BBAndPred : Preds) {
561 BasicBlock *BB = BBAndPred.first;
562 Value *Pred = BBAndPred.second;
568 PhiInserter.AddAvailableValue(BB, Pred);
569 Dominator.addAndRememberBlock(BB);
573 Term->setCondition(ParentValue);
575 if (!Dominator.resultIsRememberedBlock())
576 PhiInserter.AddAvailableValue(Dominator.result(), Default);
578 Term->setCondition(PhiInserter.GetValueInMiddleOfBlock(Parent));
583 /// Remove all PHI values coming from "From" into "To" and remember
584 /// them in DeletedPhis
585 void StructurizeCFG::delPhiValues(BasicBlock *From, BasicBlock *To) {
586 PhiMap &Map = DeletedPhis[To];
587 for (PHINode &Phi : To->phis()) {
588 while (Phi.getBasicBlockIndex(From) != -1) {
589 Value *Deleted = Phi.removeIncomingValue(From, false);
590 Map[&Phi].push_back(std::make_pair(From, Deleted));
595 /// Add a dummy PHI value as soon as we knew the new predecessor
596 void StructurizeCFG::addPhiValues(BasicBlock *From, BasicBlock *To) {
597 for (PHINode &Phi : To->phis()) {
598 Value *Undef = UndefValue::get(Phi.getType());
599 Phi.addIncoming(Undef, From);
601 AddedPhis[To].push_back(From);
604 /// Add the real PHI value as soon as everything is set up
605 void StructurizeCFG::setPhiValues() {
606 SmallVector<PHINode *, 8> InsertedPhis;
607 SSAUpdater Updater(&InsertedPhis);
608 for (const auto &AddedPhi : AddedPhis) {
609 BasicBlock *To = AddedPhi.first;
610 const BBVector &From = AddedPhi.second;
612 if (!DeletedPhis.count(To))
615 PhiMap &Map = DeletedPhis[To];
616 for (const auto &PI : Map) {
617 PHINode *Phi = PI.first;
618 Value *Undef = UndefValue::get(Phi->getType());
619 Updater.Initialize(Phi->getType(), "");
620 Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
621 Updater.AddAvailableValue(To, Undef);
623 NearestCommonDominator Dominator(DT);
624 Dominator.addBlock(To);
625 for (const auto &VI : PI.second) {
626 Updater.AddAvailableValue(VI.first, VI.second);
627 Dominator.addAndRememberBlock(VI.first);
630 if (!Dominator.resultIsRememberedBlock())
631 Updater.AddAvailableValue(Dominator.result(), Undef);
633 for (BasicBlock *FI : From)
634 Phi->setIncomingValueForBlock(FI, Updater.GetValueAtEndOfBlock(FI));
637 DeletedPhis.erase(To);
639 assert(DeletedPhis.empty());
641 // Simplify any phis inserted by the SSAUpdater if possible
646 SimplifyQuery Q(Func->getParent()->getDataLayout());
648 for (size_t i = 0; i < InsertedPhis.size(); ++i) {
649 PHINode *Phi = InsertedPhis[i];
650 if (Value *V = SimplifyInstruction(Phi, Q)) {
651 Phi->replaceAllUsesWith(V);
652 Phi->eraseFromParent();
653 InsertedPhis[i] = InsertedPhis.back();
654 InsertedPhis.pop_back();
662 /// Remove phi values from all successors and then remove the terminator.
663 void StructurizeCFG::killTerminator(BasicBlock *BB) {
664 Instruction *Term = BB->getTerminator();
668 for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
670 delPhiValues(BB, *SI);
673 DA->removeValue(Term);
674 Term->eraseFromParent();
677 /// Let node exit(s) point to NewExit
678 void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
679 bool IncludeDominator) {
680 if (Node->isSubRegion()) {
681 Region *SubRegion = Node->getNodeAs<Region>();
682 BasicBlock *OldExit = SubRegion->getExit();
683 BasicBlock *Dominator = nullptr;
685 // Find all the edges from the sub region to the exit
686 for (auto BBI = pred_begin(OldExit), E = pred_end(OldExit); BBI != E;) {
687 // Incrememt BBI before mucking with BB's terminator.
688 BasicBlock *BB = *BBI++;
690 if (!SubRegion->contains(BB))
693 // Modify the edges to point to the new exit
694 delPhiValues(BB, OldExit);
695 BB->getTerminator()->replaceUsesOfWith(OldExit, NewExit);
696 addPhiValues(BB, NewExit);
698 // Find the new dominator (if requested)
699 if (IncludeDominator) {
703 Dominator = DT->findNearestCommonDominator(Dominator, BB);
707 // Change the dominator (if requested)
709 DT->changeImmediateDominator(NewExit, Dominator);
711 // Update the region info
712 SubRegion->replaceExit(NewExit);
714 BasicBlock *BB = Node->getNodeAs<BasicBlock>();
716 BranchInst::Create(NewExit, BB);
717 addPhiValues(BB, NewExit);
718 if (IncludeDominator)
719 DT->changeImmediateDominator(NewExit, BB);
723 /// Create a new flow node and update dominator tree and region info
724 BasicBlock *StructurizeCFG::getNextFlow(BasicBlock *Dominator) {
725 LLVMContext &Context = Func->getContext();
726 BasicBlock *Insert = Order.empty() ? ParentRegion->getExit() :
727 Order.back()->getEntry();
728 BasicBlock *Flow = BasicBlock::Create(Context, FlowBlockName,
730 DT->addNewBlock(Flow, Dominator);
731 ParentRegion->getRegionInfo()->setRegionFor(Flow, ParentRegion);
735 /// Create a new or reuse the previous node as flow node
736 BasicBlock *StructurizeCFG::needPrefix(bool NeedEmpty) {
737 BasicBlock *Entry = PrevNode->getEntry();
739 if (!PrevNode->isSubRegion()) {
740 killTerminator(Entry);
741 if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
745 // create a new flow node
746 BasicBlock *Flow = getNextFlow(Entry);
749 changeExit(PrevNode, Flow, true);
750 PrevNode = ParentRegion->getBBNode(Flow);
754 /// Returns the region exit if possible, otherwise just a new flow node
755 BasicBlock *StructurizeCFG::needPostfix(BasicBlock *Flow,
756 bool ExitUseAllowed) {
757 if (!Order.empty() || !ExitUseAllowed)
758 return getNextFlow(Flow);
760 BasicBlock *Exit = ParentRegion->getExit();
761 DT->changeImmediateDominator(Exit, Flow);
762 addPhiValues(Flow, Exit);
766 /// Set the previous node
767 void StructurizeCFG::setPrevNode(BasicBlock *BB) {
768 PrevNode = ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB)
772 /// Does BB dominate all the predicates of Node?
773 bool StructurizeCFG::dominatesPredicates(BasicBlock *BB, RegionNode *Node) {
774 BBPredicates &Preds = Predicates[Node->getEntry()];
775 return llvm::all_of(Preds, [&](std::pair<BasicBlock *, Value *> Pred) {
776 return DT->dominates(BB, Pred.first);
780 /// Can we predict that this node will always be called?
781 bool StructurizeCFG::isPredictableTrue(RegionNode *Node) {
782 BBPredicates &Preds = Predicates[Node->getEntry()];
783 bool Dominated = false;
785 // Regionentry is always true
789 for (std::pair<BasicBlock*, Value*> Pred : Preds) {
790 BasicBlock *BB = Pred.first;
791 Value *V = Pred.second;
796 if (!Dominated && DT->dominates(BB, PrevNode->getEntry()))
800 // TODO: The dominator check is too strict
804 /// Take one node from the order vector and wire it up
805 void StructurizeCFG::wireFlow(bool ExitUseAllowed,
806 BasicBlock *LoopEnd) {
807 RegionNode *Node = Order.pop_back_val();
808 Visited.insert(Node->getEntry());
810 if (isPredictableTrue(Node)) {
811 // Just a linear flow
813 changeExit(PrevNode, Node->getEntry(), true);
817 // Insert extra prefix node (or reuse last one)
818 BasicBlock *Flow = needPrefix(false);
820 // Insert extra postfix node (or use exit instead)
821 BasicBlock *Entry = Node->getEntry();
822 BasicBlock *Next = needPostfix(Flow, ExitUseAllowed);
824 // let it point to entry and next block
825 Conditions.push_back(BranchInst::Create(Entry, Next, BoolUndef, Flow));
826 addPhiValues(Flow, Entry);
827 DT->changeImmediateDominator(Entry, Flow);
830 while (!Order.empty() && !Visited.count(LoopEnd) &&
831 dominatesPredicates(Entry, Order.back())) {
832 handleLoops(false, LoopEnd);
835 changeExit(PrevNode, Next, false);
840 void StructurizeCFG::handleLoops(bool ExitUseAllowed,
841 BasicBlock *LoopEnd) {
842 RegionNode *Node = Order.back();
843 BasicBlock *LoopStart = Node->getEntry();
845 if (!Loops.count(LoopStart)) {
846 wireFlow(ExitUseAllowed, LoopEnd);
850 if (!isPredictableTrue(Node))
851 LoopStart = needPrefix(true);
853 LoopEnd = Loops[Node->getEntry()];
854 wireFlow(false, LoopEnd);
855 while (!Visited.count(LoopEnd)) {
856 handleLoops(false, LoopEnd);
859 // If the start of the loop is the entry block, we can't branch to it so
860 // insert a new dummy entry block.
861 Function *LoopFunc = LoopStart->getParent();
862 if (LoopStart == &LoopFunc->getEntryBlock()) {
863 LoopStart->setName("entry.orig");
865 BasicBlock *NewEntry =
866 BasicBlock::Create(LoopStart->getContext(),
870 BranchInst::Create(LoopStart, NewEntry);
871 DT->setNewRoot(NewEntry);
874 // Create an extra loop end node
875 LoopEnd = needPrefix(false);
876 BasicBlock *Next = needPostfix(LoopEnd, ExitUseAllowed);
877 LoopConds.push_back(BranchInst::Create(Next, LoopStart,
878 BoolUndef, LoopEnd));
879 addPhiValues(LoopEnd, LoopStart);
883 /// After this function control flow looks like it should be, but
884 /// branches and PHI nodes only have undefined conditions.
885 void StructurizeCFG::createFlow() {
886 BasicBlock *Exit = ParentRegion->getExit();
887 bool EntryDominatesExit = DT->dominates(ParentRegion->getEntry(), Exit);
897 while (!Order.empty()) {
898 handleLoops(EntryDominatesExit, nullptr);
902 changeExit(PrevNode, Exit, EntryDominatesExit);
904 assert(EntryDominatesExit);
907 /// Handle a rare case where the disintegrated nodes instructions
908 /// no longer dominate all their uses. Not sure if this is really necessary
909 void StructurizeCFG::rebuildSSA() {
911 for (BasicBlock *BB : ParentRegion->blocks())
912 for (Instruction &I : *BB) {
913 bool Initialized = false;
914 // We may modify the use list as we iterate over it, so be careful to
915 // compute the next element in the use list at the top of the loop.
916 for (auto UI = I.use_begin(), E = I.use_end(); UI != E;) {
918 Instruction *User = cast<Instruction>(U.getUser());
919 if (User->getParent() == BB) {
921 } else if (PHINode *UserPN = dyn_cast<PHINode>(User)) {
922 if (UserPN->getIncomingBlock(U) == BB)
926 if (DT->dominates(&I, User))
930 Value *Undef = UndefValue::get(I.getType());
931 Updater.Initialize(I.getType(), "");
932 Updater.AddAvailableValue(&Func->getEntryBlock(), Undef);
933 Updater.AddAvailableValue(BB, &I);
936 Updater.RewriteUseAfterInsertions(U);
941 static bool hasOnlyUniformBranches(Region *R, unsigned UniformMDKindID,
942 const LegacyDivergenceAnalysis &DA) {
943 // Bool for if all sub-regions are uniform.
944 bool SubRegionsAreUniform = true;
945 // Count of how many direct children are conditional.
946 unsigned ConditionalDirectChildren = 0;
948 for (auto E : R->elements()) {
949 if (!E->isSubRegion()) {
950 auto Br = dyn_cast<BranchInst>(E->getEntry()->getTerminator());
951 if (!Br || !Br->isConditional())
954 if (!DA.isUniform(Br))
957 // One of our direct children is conditional.
958 ConditionalDirectChildren++;
960 LLVM_DEBUG(dbgs() << "BB: " << Br->getParent()->getName()
961 << " has uniform terminator\n");
963 // Explicitly refuse to treat regions as uniform if they have non-uniform
964 // subregions. We cannot rely on DivergenceAnalysis for branches in
965 // subregions because those branches may have been removed and re-created,
966 // so we look for our metadata instead.
968 // Warning: It would be nice to treat regions as uniform based only on
969 // their direct child basic blocks' terminators, regardless of whether
970 // subregions are uniform or not. However, this requires a very careful
971 // look at SIAnnotateControlFlow to make sure nothing breaks there.
972 for (auto BB : E->getNodeAs<Region>()->blocks()) {
973 auto Br = dyn_cast<BranchInst>(BB->getTerminator());
974 if (!Br || !Br->isConditional())
977 if (!Br->getMetadata(UniformMDKindID)) {
978 // Early exit if we cannot have relaxed uniform regions.
979 if (!RelaxedUniformRegions)
982 SubRegionsAreUniform = false;
989 // Our region is uniform if:
990 // 1. All conditional branches that are direct children are uniform (checked
993 // a. All sub-regions are uniform.
994 // b. There is one or less conditional branches among the direct children.
995 return SubRegionsAreUniform || (ConditionalDirectChildren <= 1);
998 /// Run the transformation for each region found
999 bool StructurizeCFG::runOnRegion(Region *R, RGPassManager &RGM) {
1000 if (R->isTopLevelRegion())
1005 if (SkipUniformRegions) {
1006 // TODO: We could probably be smarter here with how we handle sub-regions.
1007 // We currently rely on the fact that metadata is set by earlier invocations
1008 // of the pass on sub-regions, and that this metadata doesn't get lost --
1009 // but we shouldn't rely on metadata for correctness!
1010 unsigned UniformMDKindID =
1011 R->getEntry()->getContext().getMDKindID("structurizecfg.uniform");
1012 DA = &getAnalysis<LegacyDivergenceAnalysis>();
1014 if (hasOnlyUniformBranches(R, UniformMDKindID, *DA)) {
1015 LLVM_DEBUG(dbgs() << "Skipping region with uniform control flow: " << *R
1018 // Mark all direct child block terminators as having been treated as
1019 // uniform. To account for a possible future in which non-uniform
1020 // sub-regions are treated more cleverly, indirect children are not
1021 // marked as uniform.
1022 MDNode *MD = MDNode::get(R->getEntry()->getParent()->getContext(), {});
1023 for (RegionNode *E : R->elements()) {
1024 if (E->isSubRegion())
1027 if (Instruction *Term = E->getEntry()->getTerminator())
1028 Term->setMetadata(UniformMDKindID, MD);
1035 Func = R->getEntry()->getParent();
1038 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1039 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1044 insertConditions(false);
1045 insertConditions(true);
1052 DeletedPhis.clear();
1063 Pass *llvm::createStructurizeCFGPass(bool SkipUniformRegions) {
1064 return new StructurizeCFG(SkipUniformRegions);