1 //===-- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 // This file implements a TargetTransformInfo analysis pass specific to the
12 // AMDGPU target machine. It uses the target's detailed information to provide
13 // more precise answers to certain TTI queries, while letting the target
14 // independent and default TTI implementations handle the rest.
16 //===----------------------------------------------------------------------===//
18 #include "AMDGPUTargetTransformInfo.h"
19 #include "llvm/Analysis/LoopInfo.h"
20 #include "llvm/Analysis/TargetTransformInfo.h"
21 #include "llvm/Analysis/ValueTracking.h"
22 #include "llvm/CodeGen/BasicTTIImpl.h"
23 #include "llvm/IR/Module.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Target/CostTable.h"
27 #include "llvm/Target/TargetLowering.h"
30 #define DEBUG_TYPE "AMDGPUtti"
33 void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L,
34 TTI::UnrollingPreferences &UP) {
35 UP.Threshold = 300; // Twice the default.
36 UP.MaxCount = UINT_MAX;
39 // TODO: Do we want runtime unrolling?
41 for (const BasicBlock *BB : L->getBlocks()) {
42 const DataLayout &DL = BB->getModule()->getDataLayout();
43 for (const Instruction &I : *BB) {
44 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I);
45 if (!GEP || GEP->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS)
48 const Value *Ptr = GEP->getPointerOperand();
49 const AllocaInst *Alloca =
50 dyn_cast<AllocaInst>(GetUnderlyingObject(Ptr, DL));
52 // We want to do whatever we can to limit the number of alloca
53 // instructions that make it through to the code generator. allocas
54 // require us to use indirect addressing, which is slow and prone to
55 // compiler bugs. If this loop does an address calculation on an
56 // alloca ptr, then we want to use a higher than normal loop unroll
57 // threshold. This will give SROA a better chance to eliminate these
60 // Don't use the maximum allowed value here as it will make some
61 // programs way too big.
68 unsigned AMDGPUTTIImpl::getNumberOfRegisters(bool Vec) {
72 // Number of VGPRs on SI.
73 if (ST->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS)
76 return 4 * 128; // XXX - 4 channels. Should these count as vector instead?
79 unsigned AMDGPUTTIImpl::getRegisterBitWidth(bool Vector) {
80 return Vector ? 0 : 32;
83 unsigned AMDGPUTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) {
85 case AMDGPUAS::GLOBAL_ADDRESS:
86 case AMDGPUAS::CONSTANT_ADDRESS:
87 case AMDGPUAS::FLAT_ADDRESS:
89 case AMDGPUAS::LOCAL_ADDRESS:
90 case AMDGPUAS::REGION_ADDRESS:
92 case AMDGPUAS::PRIVATE_ADDRESS:
93 return 8 * ST->getMaxPrivateElementSize();
95 if (ST->getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS &&
96 (AddrSpace == AMDGPUAS::PARAM_D_ADDRESS ||
97 AddrSpace == AMDGPUAS::PARAM_I_ADDRESS ||
98 (AddrSpace >= AMDGPUAS::CONSTANT_BUFFER_0 &&
99 AddrSpace <= AMDGPUAS::CONSTANT_BUFFER_15)))
101 llvm_unreachable("unhandled address space");
105 unsigned AMDGPUTTIImpl::getMaxInterleaveFactor(unsigned VF) {
106 // Semi-arbitrary large amount.
110 int AMDGPUTTIImpl::getArithmeticInstrCost(
111 unsigned Opcode, Type *Ty, TTI::OperandValueKind Opd1Info,
112 TTI::OperandValueKind Opd2Info, TTI::OperandValueProperties Opd1PropInfo,
113 TTI::OperandValueProperties Opd2PropInfo) {
115 EVT OrigTy = TLI->getValueType(DL, Ty);
116 if (!OrigTy.isSimple()) {
117 return BaseT::getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
118 Opd1PropInfo, Opd2PropInfo);
121 // Legalize the type.
122 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
123 int ISD = TLI->InstructionOpcodeToISD(Opcode);
125 // Because we don't have any legal vector operations, but the legal types, we
126 // need to account for split vectors.
127 unsigned NElts = LT.second.isVector() ?
128 LT.second.getVectorNumElements() : 1;
130 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
137 return get64BitInstrCost() * LT.first * NElts;
140 return getFullRateInstrCost() * LT.first * NElts;
147 if (SLT == MVT::i64){
148 // and, or and xor are typically split into 2 VALU instructions.
149 return 2 * getFullRateInstrCost() * LT.first * NElts;
152 return LT.first * NElts * getFullRateInstrCost();
155 const int QuarterRateCost = getQuarterRateInstrCost();
156 if (SLT == MVT::i64) {
157 const int FullRateCost = getFullRateInstrCost();
158 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
162 return QuarterRateCost * NElts * LT.first;
168 return LT.first * NElts * get64BitInstrCost();
170 if (SLT == MVT::f32 || SLT == MVT::f16)
171 return LT.first * NElts * getFullRateInstrCost();
176 // FIXME: frem should be handled separately. The fdiv in it is most of it,
177 // but the current lowering is also not entirely correct.
178 if (SLT == MVT::f64) {
179 int Cost = 4 * get64BitInstrCost() + 7 * getQuarterRateInstrCost();
181 // Add cost of workaround.
182 if (ST->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS)
183 Cost += 3 * getFullRateInstrCost();
185 return LT.first * Cost * NElts;
188 // Assuming no fp32 denormals lowering.
189 if (SLT == MVT::f32 || SLT == MVT::f16) {
190 assert(!ST->hasFP32Denormals() && "will change when supported");
191 int Cost = 7 * getFullRateInstrCost() + 1 * getQuarterRateInstrCost();
192 return LT.first * NElts * Cost;
200 return BaseT::getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
201 Opd1PropInfo, Opd2PropInfo);
204 unsigned AMDGPUTTIImpl::getCFInstrCost(unsigned Opcode) {
205 // XXX - For some reason this isn't called for switch.
207 case Instruction::Br:
208 case Instruction::Ret:
211 return BaseT::getCFInstrCost(Opcode);
215 int AMDGPUTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
218 case Instruction::ExtractElement:
219 case Instruction::InsertElement:
220 // Extracts are just reads of a subregister, so are free. Inserts are
221 // considered free because we don't want to have any cost for scalarizing
222 // operations, and we don't have to copy into a different register class.
224 // Dynamic indexing isn't free and is best avoided.
225 return Index == ~0u ? 2 : 0;
227 return BaseT::getVectorInstrCost(Opcode, ValTy, Index);
231 static bool isIntrinsicSourceOfDivergence(const TargetIntrinsicInfo *TII,
232 const IntrinsicInst *I) {
233 switch (I->getIntrinsicID()) {
236 case Intrinsic::not_intrinsic:
237 // This means we have an intrinsic that isn't defined in
238 // IntrinsicsAMDGPU.td
241 case Intrinsic::amdgcn_workitem_id_x:
242 case Intrinsic::amdgcn_workitem_id_y:
243 case Intrinsic::amdgcn_workitem_id_z:
244 case Intrinsic::amdgcn_interp_p1:
245 case Intrinsic::amdgcn_interp_p2:
246 case Intrinsic::amdgcn_mbcnt_hi:
247 case Intrinsic::amdgcn_mbcnt_lo:
248 case Intrinsic::r600_read_tidig_x:
249 case Intrinsic::r600_read_tidig_y:
250 case Intrinsic::r600_read_tidig_z:
251 case Intrinsic::amdgcn_image_atomic_swap:
252 case Intrinsic::amdgcn_image_atomic_add:
253 case Intrinsic::amdgcn_image_atomic_sub:
254 case Intrinsic::amdgcn_image_atomic_smin:
255 case Intrinsic::amdgcn_image_atomic_umin:
256 case Intrinsic::amdgcn_image_atomic_smax:
257 case Intrinsic::amdgcn_image_atomic_umax:
258 case Intrinsic::amdgcn_image_atomic_and:
259 case Intrinsic::amdgcn_image_atomic_or:
260 case Intrinsic::amdgcn_image_atomic_xor:
261 case Intrinsic::amdgcn_image_atomic_inc:
262 case Intrinsic::amdgcn_image_atomic_dec:
263 case Intrinsic::amdgcn_image_atomic_cmpswap:
264 case Intrinsic::amdgcn_buffer_atomic_swap:
265 case Intrinsic::amdgcn_buffer_atomic_add:
266 case Intrinsic::amdgcn_buffer_atomic_sub:
267 case Intrinsic::amdgcn_buffer_atomic_smin:
268 case Intrinsic::amdgcn_buffer_atomic_umin:
269 case Intrinsic::amdgcn_buffer_atomic_smax:
270 case Intrinsic::amdgcn_buffer_atomic_umax:
271 case Intrinsic::amdgcn_buffer_atomic_and:
272 case Intrinsic::amdgcn_buffer_atomic_or:
273 case Intrinsic::amdgcn_buffer_atomic_xor:
274 case Intrinsic::amdgcn_buffer_atomic_cmpswap:
275 case Intrinsic::amdgcn_ps_live:
279 StringRef Name = I->getCalledFunction()->getName();
280 switch (TII->lookupName((const char *)Name.bytes_begin(), Name.size())) {
283 case AMDGPUIntrinsic::SI_fs_interp:
284 case AMDGPUIntrinsic::SI_fs_constant:
289 static bool isArgPassedInSGPR(const Argument *A) {
290 const Function *F = A->getParent();
292 // Arguments to compute shaders are never a source of divergence.
293 if (!AMDGPU::isShader(F->getCallingConv()))
296 // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
297 if (F->getAttributes().hasAttribute(A->getArgNo() + 1, Attribute::InReg) ||
298 F->getAttributes().hasAttribute(A->getArgNo() + 1, Attribute::ByVal))
301 // Everything else is in VGPRs.
306 /// \returns true if the result of the value could potentially be
307 /// different across workitems in a wavefront.
308 bool AMDGPUTTIImpl::isSourceOfDivergence(const Value *V) const {
310 if (const Argument *A = dyn_cast<Argument>(V))
311 return !isArgPassedInSGPR(A);
313 // Loads from the private address space are divergent, because threads
314 // can execute the load instruction with the same inputs and get different
317 // All other loads are not divergent, because if threads issue loads with the
318 // same arguments, they will always get the same result.
319 if (const LoadInst *Load = dyn_cast<LoadInst>(V))
320 return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS;
322 // Atomics are divergent because they are executed sequentially: when an
323 // atomic operation refers to the same address in each thread, then each
324 // thread after the first sees the value written by the previous thread as
326 if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V))
329 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
330 const TargetMachine &TM = getTLI()->getTargetMachine();
331 return isIntrinsicSourceOfDivergence(TM.getIntrinsicInfo(), Intrinsic);
334 // Assume all function calls are a source of divergence.
335 if (isa<CallInst>(V) || isa<InvokeInst>(V))