1 //===-- AMDGPUAnnotateUniformValues.cpp - ---------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// This pass adds amdgpu.uniform metadata to IR values so this information
12 /// can be used during instruction selection.
14 //===----------------------------------------------------------------------===//
17 #include "AMDGPUIntrinsicInfo.h"
18 #include "llvm/ADT/SetVector.h"
19 #include "llvm/Analysis/DivergenceAnalysis.h"
20 #include "llvm/Analysis/LoopInfo.h"
21 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
22 #include "llvm/IR/InstVisitor.h"
23 #include "llvm/IR/IRBuilder.h"
24 #include "llvm/Support/Debug.h"
25 #include "llvm/Support/raw_ostream.h"
27 #define DEBUG_TYPE "amdgpu-annotate-uniform"
33 class AMDGPUAnnotateUniformValues : public FunctionPass,
34 public InstVisitor<AMDGPUAnnotateUniformValues> {
35 DivergenceAnalysis *DA;
36 MemoryDependenceResults *MDR;
38 DenseMap<Value*, GetElementPtrInst*> noClobberClones;
43 AMDGPUAnnotateUniformValues() :
45 bool doInitialization(Module &M) override;
46 bool runOnFunction(Function &F) override;
47 StringRef getPassName() const override {
48 return "AMDGPU Annotate Uniform Values";
50 void getAnalysisUsage(AnalysisUsage &AU) const override {
51 AU.addRequired<DivergenceAnalysis>();
52 AU.addRequired<MemoryDependenceWrapperPass>();
53 AU.addRequired<LoopInfoWrapperPass>();
57 void visitBranchInst(BranchInst &I);
58 void visitLoadInst(LoadInst &I);
59 bool isClobberedInFunction(LoadInst * Load);
62 } // End anonymous namespace
64 INITIALIZE_PASS_BEGIN(AMDGPUAnnotateUniformValues, DEBUG_TYPE,
65 "Add AMDGPU uniform metadata", false, false)
66 INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
67 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
68 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
69 INITIALIZE_PASS_END(AMDGPUAnnotateUniformValues, DEBUG_TYPE,
70 "Add AMDGPU uniform metadata", false, false)
72 char AMDGPUAnnotateUniformValues::ID = 0;
74 static void setUniformMetadata(Instruction *I) {
75 I->setMetadata("amdgpu.uniform", MDNode::get(I->getContext(), {}));
77 static void setNoClobberMetadata(Instruction *I) {
78 I->setMetadata("amdgpu.noclobber", MDNode::get(I->getContext(), {}));
81 static void DFS(BasicBlock *Root, SetVector<BasicBlock*> & Set) {
82 for (auto I : predecessors(Root))
87 bool AMDGPUAnnotateUniformValues::isClobberedInFunction(LoadInst * Load) {
88 // 1. get Loop for the Load->getparent();
89 // 2. if it exists, collect all the BBs from the most outer
90 // loop and check for the writes. If NOT - start DFS over all preds.
91 // 3. Start DFS over all preds from the most outer loop header.
92 SetVector<BasicBlock *> Checklist;
93 BasicBlock *Start = Load->getParent();
94 Checklist.insert(Start);
95 const Value *Ptr = Load->getPointerOperand();
96 const Loop *L = LI->getLoopFor(Start);
101 P = P->getParentLoop();
103 Checklist.insert(L->block_begin(), L->block_end());
104 Start = L->getHeader();
107 DFS(Start, Checklist);
108 for (auto &BB : Checklist) {
109 BasicBlock::iterator StartIt = (BB == Load->getParent()) ?
110 BasicBlock::iterator(Load) : BB->end();
111 if (MDR->getPointerDependencyFrom(MemoryLocation(Ptr),
112 true, StartIt, BB, Load).isClobber())
118 void AMDGPUAnnotateUniformValues::visitBranchInst(BranchInst &I) {
119 if (I.isUnconditional())
122 Value *Cond = I.getCondition();
123 if (!DA->isUniform(Cond))
126 setUniformMetadata(I.getParent()->getTerminator());
129 void AMDGPUAnnotateUniformValues::visitLoadInst(LoadInst &I) {
130 Value *Ptr = I.getPointerOperand();
131 if (!DA->isUniform(Ptr))
133 auto isGlobalLoad = [](LoadInst &Load)->bool {
134 return Load.getPointerAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
136 // We're tracking up to the Function boundaries
137 // We cannot go beyond because of FunctionPass restrictions
138 // Thus we can ensure that memory not clobbered for memory
139 // operations that live in kernel only.
140 bool NotClobbered = isKernelFunc && !isClobberedInFunction(&I);
141 Instruction *PtrI = dyn_cast<Instruction>(Ptr);
142 if (!PtrI && NotClobbered && isGlobalLoad(I)) {
143 if (isa<Argument>(Ptr) || isa<GlobalValue>(Ptr)) {
144 // Lookup for the existing GEP
145 if (noClobberClones.count(Ptr)) {
146 PtrI = noClobberClones[Ptr];
148 // Create GEP of the Value
149 Function *F = I.getParent()->getParent();
150 Value *Idx = Constant::getIntegerValue(
151 Type::getInt32Ty(Ptr->getContext()), APInt(64, 0));
152 // Insert GEP at the entry to make it dominate all uses
153 PtrI = GetElementPtrInst::Create(
154 Ptr->getType()->getPointerElementType(), Ptr,
155 ArrayRef<Value*>(Idx), Twine(""), F->getEntryBlock().getFirstNonPHI());
157 I.replaceUsesOfWith(Ptr, PtrI);
162 setUniformMetadata(PtrI);
164 setNoClobberMetadata(PtrI);
168 bool AMDGPUAnnotateUniformValues::doInitialization(Module &M) {
172 bool AMDGPUAnnotateUniformValues::runOnFunction(Function &F) {
176 DA = &getAnalysis<DivergenceAnalysis>();
177 MDR = &getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
178 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
179 isKernelFunc = F.getCallingConv() == CallingConv::AMDGPU_KERNEL;
182 noClobberClones.clear();
187 llvm::createAMDGPUAnnotateUniformValues() {
188 return new AMDGPUAnnotateUniformValues();