1 //===--- ExpandReductions.cpp - Expand experimental reduction intrinsics --===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass implements IR expansion for reduction intrinsics, allowing targets
11 // to enable the experimental intrinsics until just before codegen.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/CodeGen/ExpandReductions.h"
16 #include "llvm/Analysis/TargetTransformInfo.h"
17 #include "llvm/CodeGen/Passes.h"
18 #include "llvm/IR/Function.h"
19 #include "llvm/IR/IRBuilder.h"
20 #include "llvm/IR/InstIterator.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Module.h"
24 #include "llvm/Pass.h"
25 #include "llvm/Transforms/Utils/LoopUtils.h"
31 unsigned getOpcode(Intrinsic::ID ID) {
33 case Intrinsic::experimental_vector_reduce_fadd:
34 return Instruction::FAdd;
35 case Intrinsic::experimental_vector_reduce_fmul:
36 return Instruction::FMul;
37 case Intrinsic::experimental_vector_reduce_add:
38 return Instruction::Add;
39 case Intrinsic::experimental_vector_reduce_mul:
40 return Instruction::Mul;
41 case Intrinsic::experimental_vector_reduce_and:
42 return Instruction::And;
43 case Intrinsic::experimental_vector_reduce_or:
44 return Instruction::Or;
45 case Intrinsic::experimental_vector_reduce_xor:
46 return Instruction::Xor;
47 case Intrinsic::experimental_vector_reduce_smax:
48 case Intrinsic::experimental_vector_reduce_smin:
49 case Intrinsic::experimental_vector_reduce_umax:
50 case Intrinsic::experimental_vector_reduce_umin:
51 return Instruction::ICmp;
52 case Intrinsic::experimental_vector_reduce_fmax:
53 case Intrinsic::experimental_vector_reduce_fmin:
54 return Instruction::FCmp;
56 llvm_unreachable("Unexpected ID");
60 RecurrenceDescriptor::MinMaxRecurrenceKind getMRK(Intrinsic::ID ID) {
62 case Intrinsic::experimental_vector_reduce_smax:
63 return RecurrenceDescriptor::MRK_SIntMax;
64 case Intrinsic::experimental_vector_reduce_smin:
65 return RecurrenceDescriptor::MRK_SIntMin;
66 case Intrinsic::experimental_vector_reduce_umax:
67 return RecurrenceDescriptor::MRK_UIntMax;
68 case Intrinsic::experimental_vector_reduce_umin:
69 return RecurrenceDescriptor::MRK_UIntMin;
70 case Intrinsic::experimental_vector_reduce_fmax:
71 return RecurrenceDescriptor::MRK_FloatMax;
72 case Intrinsic::experimental_vector_reduce_fmin:
73 return RecurrenceDescriptor::MRK_FloatMin;
75 return RecurrenceDescriptor::MRK_Invalid;
79 bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
81 SmallVector<IntrinsicInst*, 4> Worklist;
82 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I)
83 if (auto II = dyn_cast<IntrinsicInst>(&*I))
84 Worklist.push_back(II);
86 for (auto *II : Worklist) {
87 IRBuilder<> Builder(II);
89 auto ID = II->getIntrinsicID();
90 auto MRK = RecurrenceDescriptor::MRK_Invalid;
92 case Intrinsic::experimental_vector_reduce_fadd:
93 case Intrinsic::experimental_vector_reduce_fmul:
94 // FMFs must be attached to the call, otherwise it's an ordered reduction
95 // and it can't be handled by generating this shuffle sequence.
96 // TODO: Implement scalarization of ordered reductions here for targets
97 // without native support.
98 if (!II->getFastMathFlags().unsafeAlgebra())
100 Vec = II->getArgOperand(1);
102 case Intrinsic::experimental_vector_reduce_add:
103 case Intrinsic::experimental_vector_reduce_mul:
104 case Intrinsic::experimental_vector_reduce_and:
105 case Intrinsic::experimental_vector_reduce_or:
106 case Intrinsic::experimental_vector_reduce_xor:
107 case Intrinsic::experimental_vector_reduce_smax:
108 case Intrinsic::experimental_vector_reduce_smin:
109 case Intrinsic::experimental_vector_reduce_umax:
110 case Intrinsic::experimental_vector_reduce_umin:
111 case Intrinsic::experimental_vector_reduce_fmax:
112 case Intrinsic::experimental_vector_reduce_fmin:
113 Vec = II->getArgOperand(0);
119 if (!TTI->shouldExpandReduction(II))
121 auto Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK);
122 II->replaceAllUsesWith(Rdx);
123 II->eraseFromParent();
129 class ExpandReductions : public FunctionPass {
132 ExpandReductions() : FunctionPass(ID) {
133 initializeExpandReductionsPass(*PassRegistry::getPassRegistry());
136 bool runOnFunction(Function &F) override {
137 const auto *TTI =&getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
138 return expandReductions(F, TTI);
141 void getAnalysisUsage(AnalysisUsage &AU) const override {
142 AU.addRequired<TargetTransformInfoWrapperPass>();
143 AU.setPreservesCFG();
148 char ExpandReductions::ID;
149 INITIALIZE_PASS_BEGIN(ExpandReductions, "expand-reductions",
150 "Expand reduction intrinsics", false, false)
151 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
152 INITIALIZE_PASS_END(ExpandReductions, "expand-reductions",
153 "Expand reduction intrinsics", false, false)
155 FunctionPass *llvm::createExpandReductionsPass() {
156 return new ExpandReductions();
159 PreservedAnalyses ExpandReductionsPass::run(Function &F,
160 FunctionAnalysisManager &AM) {
161 const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
162 if (!expandReductions(F, &TTI))
163 return PreservedAnalyses::all();
164 PreservedAnalyses PA;
165 PA.preserveSet<CFGAnalyses>();