1 //===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// This pass does misc. AMDGPU optimizations on IR before instruction
14 //===----------------------------------------------------------------------===//
17 #include "AMDGPUIntrinsicInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "AMDGPUTargetMachine.h"
21 #include "llvm/Analysis/DivergenceAnalysis.h"
22 #include "llvm/CodeGen/Passes.h"
23 #include "llvm/IR/InstVisitor.h"
24 #include "llvm/IR/IRBuilder.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/raw_ostream.h"
28 #define DEBUG_TYPE "amdgpu-codegenprepare"
34 class AMDGPUCodeGenPrepare : public FunctionPass,
35 public InstVisitor<AMDGPUCodeGenPrepare, bool> {
36 const GCNTargetMachine *TM;
37 const SISubtarget *ST;
38 DivergenceAnalysis *DA;
42 /// \brief Copies exact/nsw/nuw flags (if any) from binary operation \p I to
43 /// binary operation \p V.
45 /// \returns Binary operation \p V.
46 Value *copyFlags(const BinaryOperator &I, Value *V) const;
48 /// \returns \p T's base element bit width.
49 unsigned getBaseElementBitWidth(const Type *T) const;
51 /// \returns Equivalent 32 bit integer type for given type \p T. For example,
52 /// if \p T is i7, then i32 is returned; if \p T is <3 x i12>, then <3 x i32>
54 Type *getI32Ty(IRBuilder<> &B, const Type *T) const;
56 /// \returns True if binary operation \p I is a signed binary operation, false
58 bool isSigned(const BinaryOperator &I) const;
60 /// \returns True if the condition of 'select' operation \p I comes from a
61 /// signed 'icmp' operation, false otherwise.
62 bool isSigned(const SelectInst &I) const;
64 /// \returns True if type \p T needs to be promoted to 32 bit integer type,
66 bool needsPromotionToI32(const Type *T) const;
68 /// \brief Promotes uniform binary operation \p I to equivalent 32 bit binary
71 /// \details \p I's base element bit width must be greater than 1 and less
72 /// than or equal 16. Promotion is done by sign or zero extending operands to
73 /// 32 bits, replacing \p I with equivalent 32 bit binary operation, and
74 /// truncating the result of 32 bit binary operation back to \p I's original
75 /// type. Division operation is not promoted.
77 /// \returns True if \p I is promoted to equivalent 32 bit binary operation,
79 bool promoteUniformOpToI32(BinaryOperator &I) const;
81 /// \brief Promotes uniform 'icmp' operation \p I to 32 bit 'icmp' operation.
83 /// \details \p I's base element bit width must be greater than 1 and less
84 /// than or equal 16. Promotion is done by sign or zero extending operands to
85 /// 32 bits, and replacing \p I with 32 bit 'icmp' operation.
88 bool promoteUniformOpToI32(ICmpInst &I) const;
90 /// \brief Promotes uniform 'select' operation \p I to 32 bit 'select'
93 /// \details \p I's base element bit width must be greater than 1 and less
94 /// than or equal 16. Promotion is done by sign or zero extending operands to
95 /// 32 bits, replacing \p I with 32 bit 'select' operation, and truncating the
96 /// result of 32 bit 'select' operation back to \p I's original type.
99 bool promoteUniformOpToI32(SelectInst &I) const;
101 /// \brief Promotes uniform 'bitreverse' intrinsic \p I to 32 bit 'bitreverse'
104 /// \details \p I's base element bit width must be greater than 1 and less
105 /// than or equal 16. Promotion is done by zero extending the operand to 32
106 /// bits, replacing \p I with 32 bit 'bitreverse' intrinsic, shifting the
107 /// result of 32 bit 'bitreverse' intrinsic to the right with zero fill (the
108 /// shift amount is 32 minus \p I's base element bit width), and truncating
109 /// the result of the shift operation back to \p I's original type.
112 bool promoteUniformBitreverseToI32(IntrinsicInst &I) const;
116 AMDGPUCodeGenPrepare(const TargetMachine *TM = nullptr) :
118 TM(static_cast<const GCNTargetMachine *>(TM)),
122 HasUnsafeFPMath(false) { }
124 bool visitFDiv(BinaryOperator &I);
126 bool visitInstruction(Instruction &I) { return false; }
127 bool visitBinaryOperator(BinaryOperator &I);
128 bool visitICmpInst(ICmpInst &I);
129 bool visitSelectInst(SelectInst &I);
131 bool visitIntrinsicInst(IntrinsicInst &I);
132 bool visitBitreverseIntrinsicInst(IntrinsicInst &I);
134 bool doInitialization(Module &M) override;
135 bool runOnFunction(Function &F) override;
137 StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
139 void getAnalysisUsage(AnalysisUsage &AU) const override {
140 AU.addRequired<DivergenceAnalysis>();
141 AU.setPreservesAll();
145 } // End anonymous namespace
147 Value *AMDGPUCodeGenPrepare::copyFlags(
148 const BinaryOperator &I, Value *V) const {
149 BinaryOperator *BinOp = dyn_cast<BinaryOperator>(V);
150 if (!BinOp) // Possibly constant expression.
153 if (isa<OverflowingBinaryOperator>(BinOp)) {
154 BinOp->setHasNoSignedWrap(I.hasNoSignedWrap());
155 BinOp->setHasNoUnsignedWrap(I.hasNoUnsignedWrap());
156 } else if (isa<PossiblyExactOperator>(BinOp))
157 BinOp->setIsExact(I.isExact());
162 unsigned AMDGPUCodeGenPrepare::getBaseElementBitWidth(const Type *T) const {
163 assert(needsPromotionToI32(T) && "T does not need promotion to i32");
165 if (T->isIntegerTy())
166 return T->getIntegerBitWidth();
167 return cast<VectorType>(T)->getElementType()->getIntegerBitWidth();
170 Type *AMDGPUCodeGenPrepare::getI32Ty(IRBuilder<> &B, const Type *T) const {
171 assert(needsPromotionToI32(T) && "T does not need promotion to i32");
173 if (T->isIntegerTy())
174 return B.getInt32Ty();
175 return VectorType::get(B.getInt32Ty(), cast<VectorType>(T)->getNumElements());
178 bool AMDGPUCodeGenPrepare::isSigned(const BinaryOperator &I) const {
179 return I.getOpcode() == Instruction::AShr ||
180 I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::SRem;
183 bool AMDGPUCodeGenPrepare::isSigned(const SelectInst &I) const {
184 return isa<ICmpInst>(I.getOperand(0)) ?
185 cast<ICmpInst>(I.getOperand(0))->isSigned() : false;
188 bool AMDGPUCodeGenPrepare::needsPromotionToI32(const Type *T) const {
189 if (T->isIntegerTy() && T->getIntegerBitWidth() > 1 &&
190 T->getIntegerBitWidth() <= 16)
192 if (!T->isVectorTy())
194 return needsPromotionToI32(cast<VectorType>(T)->getElementType());
197 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(BinaryOperator &I) const {
198 assert(needsPromotionToI32(I.getType()) &&
199 "I does not need promotion to i32");
201 if (I.getOpcode() == Instruction::SDiv ||
202 I.getOpcode() == Instruction::UDiv)
205 IRBuilder<> Builder(&I);
206 Builder.SetCurrentDebugLocation(I.getDebugLoc());
208 Type *I32Ty = getI32Ty(Builder, I.getType());
209 Value *ExtOp0 = nullptr;
210 Value *ExtOp1 = nullptr;
211 Value *ExtRes = nullptr;
212 Value *TruncRes = nullptr;
215 ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty);
216 ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
218 ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty);
219 ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
221 ExtRes = copyFlags(I, Builder.CreateBinOp(I.getOpcode(), ExtOp0, ExtOp1));
222 TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
224 I.replaceAllUsesWith(TruncRes);
230 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(ICmpInst &I) const {
231 assert(needsPromotionToI32(I.getOperand(0)->getType()) &&
232 "I does not need promotion to i32");
234 IRBuilder<> Builder(&I);
235 Builder.SetCurrentDebugLocation(I.getDebugLoc());
237 Type *I32Ty = getI32Ty(Builder, I.getOperand(0)->getType());
238 Value *ExtOp0 = nullptr;
239 Value *ExtOp1 = nullptr;
240 Value *NewICmp = nullptr;
243 ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty);
244 ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
246 ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty);
247 ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
249 NewICmp = Builder.CreateICmp(I.getPredicate(), ExtOp0, ExtOp1);
251 I.replaceAllUsesWith(NewICmp);
257 bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(SelectInst &I) const {
258 assert(needsPromotionToI32(I.getType()) &&
259 "I does not need promotion to i32");
261 IRBuilder<> Builder(&I);
262 Builder.SetCurrentDebugLocation(I.getDebugLoc());
264 Type *I32Ty = getI32Ty(Builder, I.getType());
265 Value *ExtOp1 = nullptr;
266 Value *ExtOp2 = nullptr;
267 Value *ExtRes = nullptr;
268 Value *TruncRes = nullptr;
271 ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty);
272 ExtOp2 = Builder.CreateSExt(I.getOperand(2), I32Ty);
274 ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty);
275 ExtOp2 = Builder.CreateZExt(I.getOperand(2), I32Ty);
277 ExtRes = Builder.CreateSelect(I.getOperand(0), ExtOp1, ExtOp2);
278 TruncRes = Builder.CreateTrunc(ExtRes, I.getType());
280 I.replaceAllUsesWith(TruncRes);
286 bool AMDGPUCodeGenPrepare::promoteUniformBitreverseToI32(
287 IntrinsicInst &I) const {
288 assert(I.getIntrinsicID() == Intrinsic::bitreverse &&
289 "I must be bitreverse intrinsic");
290 assert(needsPromotionToI32(I.getType()) &&
291 "I does not need promotion to i32");
293 IRBuilder<> Builder(&I);
294 Builder.SetCurrentDebugLocation(I.getDebugLoc());
296 Type *I32Ty = getI32Ty(Builder, I.getType());
298 Intrinsic::getDeclaration(Mod, Intrinsic::bitreverse, { I32Ty });
299 Value *ExtOp = Builder.CreateZExt(I.getOperand(0), I32Ty);
300 Value *ExtRes = Builder.CreateCall(I32, { ExtOp });
302 Builder.CreateLShr(ExtRes, 32 - getBaseElementBitWidth(I.getType()));
304 Builder.CreateTrunc(LShrOp, I.getType());
306 I.replaceAllUsesWith(TruncRes);
312 static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv) {
313 const ConstantFP *CNum = dyn_cast<ConstantFP>(Num);
317 // Reciprocal f32 is handled separately without denormals.
318 return UnsafeDiv || CNum->isExactlyValue(+1.0);
321 // Insert an intrinsic for fast fdiv for safe math situations where we can
322 // reduce precision. Leave fdiv for situations where the generic node is
323 // expected to be optimized.
324 bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) {
325 Type *Ty = FDiv.getType();
327 if (!Ty->getScalarType()->isFloatTy())
330 MDNode *FPMath = FDiv.getMetadata(LLVMContext::MD_fpmath);
334 const FPMathOperator *FPOp = cast<const FPMathOperator>(&FDiv);
335 float ULP = FPOp->getFPAccuracy();
339 FastMathFlags FMF = FPOp->getFastMathFlags();
340 bool UnsafeDiv = HasUnsafeFPMath || FMF.unsafeAlgebra() ||
341 FMF.allowReciprocal();
342 if (ST->hasFP32Denormals() && !UnsafeDiv)
345 IRBuilder<> Builder(FDiv.getParent(), std::next(FDiv.getIterator()), FPMath);
346 Builder.setFastMathFlags(FMF);
347 Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
349 const AMDGPUIntrinsicInfo *II = TM->getIntrinsicInfo();
351 = II->getDeclaration(Mod, AMDGPUIntrinsic::amdgcn_fdiv_fast, {});
353 Value *Num = FDiv.getOperand(0);
354 Value *Den = FDiv.getOperand(1);
356 Value *NewFDiv = nullptr;
358 if (VectorType *VT = dyn_cast<VectorType>(Ty)) {
359 NewFDiv = UndefValue::get(VT);
361 // FIXME: Doesn't do the right thing for cases where the vector is partially
362 // constant. This works when the scalarizer pass is run first.
363 for (unsigned I = 0, E = VT->getNumElements(); I != E; ++I) {
364 Value *NumEltI = Builder.CreateExtractElement(Num, I);
365 Value *DenEltI = Builder.CreateExtractElement(Den, I);
368 if (shouldKeepFDivF32(NumEltI, UnsafeDiv)) {
369 NewElt = Builder.CreateFDiv(NumEltI, DenEltI);
371 NewElt = Builder.CreateCall(Decl, { NumEltI, DenEltI });
374 NewFDiv = Builder.CreateInsertElement(NewFDiv, NewElt, I);
377 if (!shouldKeepFDivF32(Num, UnsafeDiv))
378 NewFDiv = Builder.CreateCall(Decl, { Num, Den });
382 FDiv.replaceAllUsesWith(NewFDiv);
383 NewFDiv->takeName(&FDiv);
384 FDiv.eraseFromParent();
390 static bool hasUnsafeFPMath(const Function &F) {
391 Attribute Attr = F.getFnAttribute("unsafe-fp-math");
392 return Attr.getValueAsString() == "true";
395 bool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) {
396 bool Changed = false;
398 if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
400 Changed |= promoteUniformOpToI32(I);
405 bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) {
406 bool Changed = false;
408 if (ST->has16BitInsts() && needsPromotionToI32(I.getOperand(0)->getType()) &&
410 Changed |= promoteUniformOpToI32(I);
415 bool AMDGPUCodeGenPrepare::visitSelectInst(SelectInst &I) {
416 bool Changed = false;
418 if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
420 Changed |= promoteUniformOpToI32(I);
425 bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) {
426 switch (I.getIntrinsicID()) {
427 case Intrinsic::bitreverse:
428 return visitBitreverseIntrinsicInst(I);
434 bool AMDGPUCodeGenPrepare::visitBitreverseIntrinsicInst(IntrinsicInst &I) {
435 bool Changed = false;
437 if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) &&
439 Changed |= promoteUniformBitreverseToI32(I);
444 bool AMDGPUCodeGenPrepare::doInitialization(Module &M) {
449 bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
450 if (!TM || skipFunction(F))
453 ST = &TM->getSubtarget<SISubtarget>(F);
454 DA = &getAnalysis<DivergenceAnalysis>();
455 HasUnsafeFPMath = hasUnsafeFPMath(F);
457 bool MadeChange = false;
459 for (BasicBlock &BB : F) {
460 BasicBlock::iterator Next;
461 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; I = Next) {
463 MadeChange |= visit(*I);
470 INITIALIZE_TM_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
471 "AMDGPU IR optimizations", false, false)
472 INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis)
473 INITIALIZE_TM_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE,
474 "AMDGPU IR optimizations", false, false)
476 char AMDGPUCodeGenPrepare::ID = 0;
478 FunctionPass *llvm::createAMDGPUCodeGenPreparePass(const GCNTargetMachine *TM) {
479 return new AMDGPUCodeGenPrepare(TM);