1 //===- CostModel.cpp ------ Cost Model Analysis ---------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the cost model analysis. It provides a very basic cost
11 // estimation for LLVM-IR. This analysis uses the services of the codegen
12 // to approximate the cost of any IR instruction when lowered to machine
13 // instructions. The cost results are unit-less and the cost number represents
14 // the throughput of the machine assuming that all loads hit the cache, all
15 // branches are predicted, etc. The cost numbers can be added in order to
16 // compare two or more transformation alternatives.
18 //===----------------------------------------------------------------------===//
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/Analysis/Passes.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/Analysis/VectorUtils.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/Instructions.h"
26 #include "llvm/IR/IntrinsicInst.h"
27 #include "llvm/IR/Value.h"
28 #include "llvm/Pass.h"
29 #include "llvm/Support/CommandLine.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/raw_ostream.h"
34 #define CM_NAME "cost-model"
35 #define DEBUG_TYPE CM_NAME
37 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
39 cl::desc("Recognize reduction patterns."));
42 class CostModelAnalysis : public FunctionPass {
45 static char ID; // Class identification, replacement for typeinfo
46 CostModelAnalysis() : FunctionPass(ID), F(nullptr), TTI(nullptr) {
47 initializeCostModelAnalysisPass(
48 *PassRegistry::getPassRegistry());
51 /// Returns the expected cost of the instruction.
52 /// Returns -1 if the cost is unknown.
53 /// Note, this method does not cache the cost calculation and it
54 /// can be expensive in some cases.
55 unsigned getInstructionCost(const Instruction *I) const;
58 void getAnalysisUsage(AnalysisUsage &AU) const override;
59 bool runOnFunction(Function &F) override;
60 void print(raw_ostream &OS, const Module*) const override;
62 /// The function that we analyze.
64 /// Target information.
65 const TargetTransformInfo *TTI;
67 } // End of anonymous namespace
69 // Register this pass.
70 char CostModelAnalysis::ID = 0;
71 static const char cm_name[] = "Cost Model Analysis";
72 INITIALIZE_PASS_BEGIN(CostModelAnalysis, CM_NAME, cm_name, false, true)
73 INITIALIZE_PASS_END (CostModelAnalysis, CM_NAME, cm_name, false, true)
75 FunctionPass *llvm::createCostModelAnalysisPass() {
76 return new CostModelAnalysis();
80 CostModelAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
85 CostModelAnalysis::runOnFunction(Function &F) {
87 auto *TTIWP = getAnalysisIfAvailable<TargetTransformInfoWrapperPass>();
88 TTI = TTIWP ? &TTIWP->getTTI(F) : nullptr;
93 static bool isReverseVectorMask(ArrayRef<int> Mask) {
94 for (unsigned i = 0, MaskSize = Mask.size(); i < MaskSize; ++i)
95 if (Mask[i] >= 0 && Mask[i] != (int)(MaskSize - 1 - i))
100 static bool isSingleSourceVectorMask(ArrayRef<int> Mask) {
103 for (unsigned i = 0, NumVecElts = Mask.size(); i < NumVecElts; ++i) {
105 if ((unsigned)Mask[i] >= NumVecElts)
111 return !(Vec0 && Vec1);
114 static bool isZeroEltBroadcastVectorMask(ArrayRef<int> Mask) {
115 for (unsigned i = 0; i < Mask.size(); ++i)
121 static bool isAlternateVectorMask(ArrayRef<int> Mask) {
122 bool isAlternate = true;
123 unsigned MaskSize = Mask.size();
125 // Example: shufflevector A, B, <0,5,2,7>
126 for (unsigned i = 0; i < MaskSize && isAlternate; ++i) {
129 isAlternate = Mask[i] == (int)((i & 1) ? MaskSize + i : i);
136 // Example: shufflevector A, B, <4,1,6,3>
137 for (unsigned i = 0; i < MaskSize && isAlternate; ++i) {
140 isAlternate = Mask[i] == (int)((i & 1) ? i : MaskSize + i);
146 static TargetTransformInfo::OperandValueKind getOperandInfo(Value *V) {
147 TargetTransformInfo::OperandValueKind OpInfo =
148 TargetTransformInfo::OK_AnyValue;
150 // Check for a splat of a constant or for a non uniform vector of constants.
151 if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {
152 OpInfo = TargetTransformInfo::OK_NonUniformConstantValue;
153 if (cast<Constant>(V)->getSplatValue() != nullptr)
154 OpInfo = TargetTransformInfo::OK_UniformConstantValue;
157 // Check for a splat of a uniform value. This is not loop aware, so return
158 // true only for the obviously uniform cases (argument, globalvalue)
159 const Value *Splat = getSplatValue(V);
160 if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat)))
161 OpInfo = TargetTransformInfo::OK_UniformValue;
166 static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
168 // We don't need a shuffle if we just want to have element 0 in position 0 of
170 if (!SI && Level == 0 && IsLeft)
175 SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1);
177 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether
178 // we look at the left or right side.
179 for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
182 SmallVector<int, 16> ActualMask = SI->getShuffleMask();
183 return Mask == ActualMask;
186 static bool matchPairwiseReductionAtLevel(const BinaryOperator *BinOp,
187 unsigned Level, unsigned NumLevels) {
188 // Match one level of pairwise operations.
189 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
190 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
191 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
192 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
193 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
194 if (BinOp == nullptr)
197 assert(BinOp->getType()->isVectorTy() && "Expecting a vector type");
199 unsigned Opcode = BinOp->getOpcode();
200 Value *L = BinOp->getOperand(0);
201 Value *R = BinOp->getOperand(1);
203 ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(L);
206 ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(R);
210 // On level 0 we can omit one shufflevector instruction.
211 if (!Level && !RS && !LS)
214 // Shuffle inputs must match.
215 Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr;
216 Value *NextLevelOpR = RS ? RS->getOperand(0) : nullptr;
217 Value *NextLevelOp = nullptr;
218 if (NextLevelOpR && NextLevelOpL) {
219 // If we have two shuffles their operands must match.
220 if (NextLevelOpL != NextLevelOpR)
223 NextLevelOp = NextLevelOpL;
224 } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) {
225 // On the first level we can omit the shufflevector <0, undef,...>. So the
226 // input to the other shufflevector <1, undef> must match with one of the
227 // inputs to the current binary operation.
229 // %NextLevelOpL = shufflevector %R, <1, undef ...>
230 // %BinOp = fadd %NextLevelOpL, %R
231 if (NextLevelOpL && NextLevelOpL != R)
233 else if (NextLevelOpR && NextLevelOpR != L)
236 NextLevelOp = NextLevelOpL ? R : L;
240 // Check that the next levels binary operation exists and matches with the
242 BinaryOperator *NextLevelBinOp = nullptr;
243 if (Level + 1 != NumLevels) {
244 if (!(NextLevelBinOp = dyn_cast<BinaryOperator>(NextLevelOp)))
246 else if (NextLevelBinOp->getOpcode() != Opcode)
250 // Shuffle mask for pairwise operation must match.
251 if (matchPairwiseShuffleMask(LS, true, Level)) {
252 if (!matchPairwiseShuffleMask(RS, false, Level))
254 } else if (matchPairwiseShuffleMask(RS, true, Level)) {
255 if (!matchPairwiseShuffleMask(LS, false, Level))
260 if (++Level == NumLevels)
264 return matchPairwiseReductionAtLevel(NextLevelBinOp, Level, NumLevels);
267 static bool matchPairwiseReduction(const ExtractElementInst *ReduxRoot,
268 unsigned &Opcode, Type *&Ty) {
269 if (!EnableReduxCost)
272 // Need to extract the first element.
273 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
276 Idx = CI->getZExtValue();
280 BinaryOperator *RdxStart = dyn_cast<BinaryOperator>(ReduxRoot->getOperand(0));
284 Type *VecTy = ReduxRoot->getOperand(0)->getType();
285 unsigned NumVecElems = VecTy->getVectorNumElements();
286 if (!isPowerOf2_32(NumVecElems))
289 // We look for a sequence of shuffle,shuffle,add triples like the following
290 // that builds a pairwise reduction tree.
293 // (X0 + X1, X2 + X3, undef, undef)
294 // ((X0 + X1) + (X2 + X3), undef, undef, undef)
296 // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
297 // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
298 // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
299 // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
300 // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
301 // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
302 // <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
303 // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
304 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
305 // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
306 // %r = extractelement <4 x float> %bin.rdx8, i32 0
307 if (!matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)))
310 Opcode = RdxStart->getOpcode();
316 static std::pair<Value *, ShuffleVectorInst *>
317 getShuffleAndOtherOprd(BinaryOperator *B) {
319 Value *L = B->getOperand(0);
320 Value *R = B->getOperand(1);
321 ShuffleVectorInst *S = nullptr;
323 if ((S = dyn_cast<ShuffleVectorInst>(L)))
324 return std::make_pair(R, S);
326 S = dyn_cast<ShuffleVectorInst>(R);
327 return std::make_pair(L, S);
330 static bool matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
331 unsigned &Opcode, Type *&Ty) {
332 if (!EnableReduxCost)
335 // Need to extract the first element.
336 ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
339 Idx = CI->getZExtValue();
343 BinaryOperator *RdxStart = dyn_cast<BinaryOperator>(ReduxRoot->getOperand(0));
346 unsigned RdxOpcode = RdxStart->getOpcode();
348 Type *VecTy = ReduxRoot->getOperand(0)->getType();
349 unsigned NumVecElems = VecTy->getVectorNumElements();
350 if (!isPowerOf2_32(NumVecElems))
353 // We look for a sequence of shuffles and adds like the following matching one
354 // fadd, shuffle vector pair at a time.
356 // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef,
357 // <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
358 // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
359 // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef,
360 // <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
361 // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
362 // %r = extractelement <4 x float> %bin.rdx8, i32 0
364 unsigned MaskStart = 1;
365 Value *RdxOp = RdxStart;
366 SmallVector<int, 32> ShuffleMask(NumVecElems, 0);
367 unsigned NumVecElemsRemain = NumVecElems;
368 while (NumVecElemsRemain - 1) {
369 // Check for the right reduction operation.
370 BinaryOperator *BinOp;
371 if (!(BinOp = dyn_cast<BinaryOperator>(RdxOp)))
373 if (BinOp->getOpcode() != RdxOpcode)
377 ShuffleVectorInst *Shuffle;
378 std::tie(NextRdxOp, Shuffle) = getShuffleAndOtherOprd(BinOp);
380 // Check the current reduction operation and the shuffle use the same value.
381 if (Shuffle == nullptr)
383 if (Shuffle->getOperand(0) != NextRdxOp)
386 // Check that shuffle masks matches.
387 for (unsigned j = 0; j != MaskStart; ++j)
388 ShuffleMask[j] = MaskStart + j;
389 // Fill the rest of the mask with -1 for undef.
390 std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
392 SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
393 if (ShuffleMask != Mask)
397 NumVecElemsRemain /= 2;
406 unsigned CostModelAnalysis::getInstructionCost(const Instruction *I) const {
410 switch (I->getOpcode()) {
411 case Instruction::GetElementPtr:
412 return TTI->getUserCost(I);
414 case Instruction::Ret:
415 case Instruction::PHI:
416 case Instruction::Br: {
417 return TTI->getCFInstrCost(I->getOpcode());
419 case Instruction::Add:
420 case Instruction::FAdd:
421 case Instruction::Sub:
422 case Instruction::FSub:
423 case Instruction::Mul:
424 case Instruction::FMul:
425 case Instruction::UDiv:
426 case Instruction::SDiv:
427 case Instruction::FDiv:
428 case Instruction::URem:
429 case Instruction::SRem:
430 case Instruction::FRem:
431 case Instruction::Shl:
432 case Instruction::LShr:
433 case Instruction::AShr:
434 case Instruction::And:
435 case Instruction::Or:
436 case Instruction::Xor: {
437 TargetTransformInfo::OperandValueKind Op1VK =
438 getOperandInfo(I->getOperand(0));
439 TargetTransformInfo::OperandValueKind Op2VK =
440 getOperandInfo(I->getOperand(1));
441 SmallVector<const Value*, 2> Operands(I->operand_values());
442 return TTI->getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK,
443 Op2VK, TargetTransformInfo::OP_None,
444 TargetTransformInfo::OP_None,
447 case Instruction::Select: {
448 const SelectInst *SI = cast<SelectInst>(I);
449 Type *CondTy = SI->getCondition()->getType();
450 return TTI->getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy);
452 case Instruction::ICmp:
453 case Instruction::FCmp: {
454 Type *ValTy = I->getOperand(0)->getType();
455 return TTI->getCmpSelInstrCost(I->getOpcode(), ValTy);
457 case Instruction::Store: {
458 const StoreInst *SI = cast<StoreInst>(I);
459 Type *ValTy = SI->getValueOperand()->getType();
460 return TTI->getMemoryOpCost(I->getOpcode(), ValTy,
462 SI->getPointerAddressSpace());
464 case Instruction::Load: {
465 const LoadInst *LI = cast<LoadInst>(I);
466 return TTI->getMemoryOpCost(I->getOpcode(), I->getType(),
468 LI->getPointerAddressSpace());
470 case Instruction::ZExt:
471 case Instruction::SExt:
472 case Instruction::FPToUI:
473 case Instruction::FPToSI:
474 case Instruction::FPExt:
475 case Instruction::PtrToInt:
476 case Instruction::IntToPtr:
477 case Instruction::SIToFP:
478 case Instruction::UIToFP:
479 case Instruction::Trunc:
480 case Instruction::FPTrunc:
481 case Instruction::BitCast:
482 case Instruction::AddrSpaceCast: {
483 Type *SrcTy = I->getOperand(0)->getType();
484 return TTI->getCastInstrCost(I->getOpcode(), I->getType(), SrcTy);
486 case Instruction::ExtractElement: {
487 const ExtractElementInst * EEI = cast<ExtractElementInst>(I);
488 ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
491 Idx = CI->getZExtValue();
493 // Try to match a reduction sequence (series of shufflevector and vector
494 // adds followed by a extractelement).
495 unsigned ReduxOpCode;
498 if (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType))
499 return TTI->getReductionCost(ReduxOpCode, ReduxType, false);
500 else if (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType))
501 return TTI->getReductionCost(ReduxOpCode, ReduxType, true);
503 return TTI->getVectorInstrCost(I->getOpcode(),
504 EEI->getOperand(0)->getType(), Idx);
506 case Instruction::InsertElement: {
507 const InsertElementInst * IE = cast<InsertElementInst>(I);
508 ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
511 Idx = CI->getZExtValue();
512 return TTI->getVectorInstrCost(I->getOpcode(),
515 case Instruction::ShuffleVector: {
516 const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
517 Type *VecTypOp0 = Shuffle->getOperand(0)->getType();
518 unsigned NumVecElems = VecTypOp0->getVectorNumElements();
519 SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
521 if (NumVecElems == Mask.size()) {
522 if (isReverseVectorMask(Mask))
523 return TTI->getShuffleCost(TargetTransformInfo::SK_Reverse, VecTypOp0,
525 if (isAlternateVectorMask(Mask))
526 return TTI->getShuffleCost(TargetTransformInfo::SK_Alternate,
527 VecTypOp0, 0, nullptr);
529 if (isZeroEltBroadcastVectorMask(Mask))
530 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast,
531 VecTypOp0, 0, nullptr);
533 if (isSingleSourceVectorMask(Mask))
534 return TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
535 VecTypOp0, 0, nullptr);
537 return TTI->getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc,
538 VecTypOp0, 0, nullptr);
543 case Instruction::Call:
544 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
545 SmallVector<Value *, 4> Args;
546 for (unsigned J = 0, JE = II->getNumArgOperands(); J != JE; ++J)
547 Args.push_back(II->getArgOperand(J));
550 if (auto *FPMO = dyn_cast<FPMathOperator>(II))
551 FMF = FPMO->getFastMathFlags();
553 return TTI->getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(),
558 // We don't have any information on this instruction.
563 void CostModelAnalysis::print(raw_ostream &OS, const Module*) const {
567 for (BasicBlock &B : *F) {
568 for (Instruction &Inst : B) {
569 unsigned Cost = getInstructionCost(&Inst);
570 if (Cost != (unsigned)-1)
571 OS << "Cost Model: Found an estimated cost of " << Cost;
573 OS << "Cost Model: Unknown cost";
575 OS << " for instruction: " << Inst << "\n";