contrib/llvm/lib/CodeGen/ExpandReductions.cpp

   1 //===--- ExpandReductions.cpp - Expand experimental reduction intrinsics --===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This pass implements IR expansion for reduction intrinsics, allowing targets
  11 // to enable the experimental intrinsics until just before codegen.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "llvm/CodeGen/ExpandReductions.h"
  16 #include "llvm/Analysis/TargetTransformInfo.h"
  17 #include "llvm/CodeGen/Passes.h"
  18 #include "llvm/IR/Function.h"
  19 #include "llvm/IR/IRBuilder.h"
  20 #include "llvm/IR/InstIterator.h"
  21 #include "llvm/IR/IntrinsicInst.h"
  22 #include "llvm/IR/Intrinsics.h"
  23 #include "llvm/IR/Module.h"
  24 #include "llvm/Pass.h"
  25 #include "llvm/Transforms/Utils/LoopUtils.h"
  26
  27 using namespace llvm;
  28
  29 namespace {
  30
  31 unsigned getOpcode(Intrinsic::ID ID) {
  32   switch (ID) {
  33   case Intrinsic::experimental_vector_reduce_fadd:
  34     return Instruction::FAdd;
  35   case Intrinsic::experimental_vector_reduce_fmul:
  36     return Instruction::FMul;
  37   case Intrinsic::experimental_vector_reduce_add:
  38     return Instruction::Add;
  39   case Intrinsic::experimental_vector_reduce_mul:
  40     return Instruction::Mul;
  41   case Intrinsic::experimental_vector_reduce_and:
  42     return Instruction::And;
  43   case Intrinsic::experimental_vector_reduce_or:
  44     return Instruction::Or;
  45   case Intrinsic::experimental_vector_reduce_xor:
  46     return Instruction::Xor;
  47   case Intrinsic::experimental_vector_reduce_smax:
  48   case Intrinsic::experimental_vector_reduce_smin:
  49   case Intrinsic::experimental_vector_reduce_umax:
  50   case Intrinsic::experimental_vector_reduce_umin:
  51     return Instruction::ICmp;
  52   case Intrinsic::experimental_vector_reduce_fmax:
  53   case Intrinsic::experimental_vector_reduce_fmin:
  54     return Instruction::FCmp;
  55   default:
  56     llvm_unreachable("Unexpected ID");
  57   }
  58 }
  59
  60 RecurrenceDescriptor::MinMaxRecurrenceKind getMRK(Intrinsic::ID ID) {
  61   switch (ID) {
  62   case Intrinsic::experimental_vector_reduce_smax:
  63     return RecurrenceDescriptor::MRK_SIntMax;
  64   case Intrinsic::experimental_vector_reduce_smin:
  65     return RecurrenceDescriptor::MRK_SIntMin;
  66   case Intrinsic::experimental_vector_reduce_umax:
  67     return RecurrenceDescriptor::MRK_UIntMax;
  68   case Intrinsic::experimental_vector_reduce_umin:
  69     return RecurrenceDescriptor::MRK_UIntMin;
  70   case Intrinsic::experimental_vector_reduce_fmax:
  71     return RecurrenceDescriptor::MRK_FloatMax;
  72   case Intrinsic::experimental_vector_reduce_fmin:
  73     return RecurrenceDescriptor::MRK_FloatMin;
  74   default:
  75     return RecurrenceDescriptor::MRK_Invalid;
  76   }
  77 }
  78
  79 bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
  80   bool Changed = false;
  81   SmallVector<IntrinsicInst*, 4> Worklist;
  82   for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I)
  83     if (auto II = dyn_cast<IntrinsicInst>(&*I))
  84       Worklist.push_back(II);
  85
  86   for (auto *II : Worklist) {
  87     IRBuilder<> Builder(II);
  88     Value *Vec = nullptr;
  89     auto ID = II->getIntrinsicID();
  90     auto MRK = RecurrenceDescriptor::MRK_Invalid;
  91     switch (ID) {
  92     case Intrinsic::experimental_vector_reduce_fadd:
  93     case Intrinsic::experimental_vector_reduce_fmul:
  94       // FMFs must be attached to the call, otherwise it's an ordered reduction
  95       // and it can't be handled by generating this shuffle sequence.
  96       // TODO: Implement scalarization of ordered reductions here for targets
  97       // without native support.
  98       if (!II->getFastMathFlags().isFast())
  99         continue;
 100       Vec = II->getArgOperand(1);
 101       break;
 102     case Intrinsic::experimental_vector_reduce_add:
 103     case Intrinsic::experimental_vector_reduce_mul:
 104     case Intrinsic::experimental_vector_reduce_and:
 105     case Intrinsic::experimental_vector_reduce_or:
 106     case Intrinsic::experimental_vector_reduce_xor:
 107     case Intrinsic::experimental_vector_reduce_smax:
 108     case Intrinsic::experimental_vector_reduce_smin:
 109     case Intrinsic::experimental_vector_reduce_umax:
 110     case Intrinsic::experimental_vector_reduce_umin:
 111     case Intrinsic::experimental_vector_reduce_fmax:
 112     case Intrinsic::experimental_vector_reduce_fmin:
 113       Vec = II->getArgOperand(0);
 114       MRK = getMRK(ID);
 115       break;
 116     default:
 117       continue;
 118     }
 119     if (!TTI->shouldExpandReduction(II))
 120       continue;
 121     auto Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
 122     II->replaceAllUsesWith(Rdx);
 123     II->eraseFromParent();
 124     Changed = true;
 125   }
 126   return Changed;
 127 }
 128
 129 class ExpandReductions : public FunctionPass {
 130 public:
 131   static char ID;
 132   ExpandReductions() : FunctionPass(ID) {
 133     initializeExpandReductionsPass(*PassRegistry::getPassRegistry());
 134   }
 135
 136   bool runOnFunction(Function &F) override {
 137     const auto *TTI =&getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
 138     return expandReductions(F, TTI);
 139   }
 140
 141   void getAnalysisUsage(AnalysisUsage &AU) const override {
 142     AU.addRequired<TargetTransformInfoWrapperPass>();
 143     AU.setPreservesCFG();
 144   }
 145 };
 146 }
 147
 148 char ExpandReductions::ID;
 149 INITIALIZE_PASS_BEGIN(ExpandReductions, "expand-reductions",
 150                       "Expand reduction intrinsics", false, false)
 151 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 152 INITIALIZE_PASS_END(ExpandReductions, "expand-reductions",
 153                     "Expand reduction intrinsics", false, false)
 154
 155 FunctionPass *llvm::createExpandReductionsPass() {
 156   return new ExpandReductions();
 157 }
 158
 159 PreservedAnalyses ExpandReductionsPass::run(Function &F,
 160                                             FunctionAnalysisManager &AM) {
 161   const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
 162   if (!expandReductions(F, &TTI))
 163     return PreservedAnalyses::all();
 164   PreservedAnalyses PA;
 165   PA.preserveSet<CFGAnalyses>();
 166   return PA;
 167 }