1 //===----- SVEIntrinsicOpts - SVE ACLE Intrinsics Opts --------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Performs general IR level optimizations on SVE intrinsics.
12 // The main goal of this pass is to remove unnecessary reinterpret
13 // intrinsics (llvm.aarch64.sve.convert.[to|from].svbool), e.g:
15 // %1 = @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %a)
16 // %2 = @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %1)
18 // This pass also looks for ptest intrinsics & phi instructions where the
19 // operands are being needlessly converted to and from svbool_t.
21 //===----------------------------------------------------------------------===//
23 #include "Utils/AArch64BaseInfo.h"
24 #include "llvm/ADT/PostOrderIterator.h"
25 #include "llvm/ADT/SetVector.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/Dominators.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/Instructions.h"
30 #include "llvm/IR/IntrinsicInst.h"
31 #include "llvm/IR/IntrinsicsAArch64.h"
32 #include "llvm/IR/LLVMContext.h"
33 #include "llvm/IR/PatternMatch.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/Support/Debug.h"
38 using namespace llvm::PatternMatch;
40 #define DEBUG_TYPE "sve-intrinsic-opts"
43 void initializeSVEIntrinsicOptsPass(PassRegistry &);
47 struct SVEIntrinsicOpts : public ModulePass {
48 static char ID; // Pass identification, replacement for typeid
49 SVEIntrinsicOpts() : ModulePass(ID) {
50 initializeSVEIntrinsicOptsPass(*PassRegistry::getPassRegistry());
53 bool runOnModule(Module &M) override;
54 void getAnalysisUsage(AnalysisUsage &AU) const override;
57 static IntrinsicInst *isReinterpretToSVBool(Value *V);
59 static bool optimizeIntrinsic(Instruction *I);
61 bool optimizeFunctions(SmallSetVector<Function *, 4> &Functions);
63 static bool optimizeConvertFromSVBool(IntrinsicInst *I);
64 static bool optimizePTest(IntrinsicInst *I);
66 static bool processPhiNode(IntrinsicInst *I);
68 } // end anonymous namespace
70 void SVEIntrinsicOpts::getAnalysisUsage(AnalysisUsage &AU) const {
71 AU.addRequired<DominatorTreeWrapperPass>();
75 char SVEIntrinsicOpts::ID = 0;
76 static const char *name = "SVE intrinsics optimizations";
77 INITIALIZE_PASS_BEGIN(SVEIntrinsicOpts, DEBUG_TYPE, name, false, false)
78 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
79 INITIALIZE_PASS_END(SVEIntrinsicOpts, DEBUG_TYPE, name, false, false)
82 ModulePass *createSVEIntrinsicOptsPass() { return new SVEIntrinsicOpts(); }
85 /// Returns V if it's a cast from <n x 16 x i1> (aka svbool_t), nullptr
87 IntrinsicInst *SVEIntrinsicOpts::isReinterpretToSVBool(Value *V) {
88 IntrinsicInst *I = dyn_cast<IntrinsicInst>(V);
92 if (I->getIntrinsicID() != Intrinsic::aarch64_sve_convert_to_svbool)
98 /// The function will remove redundant reinterprets casting in the presence
99 /// of the control flow
100 bool SVEIntrinsicOpts::processPhiNode(IntrinsicInst *X) {
102 SmallVector<Instruction *, 32> Worklist;
103 auto RequiredType = X->getType();
105 auto *PN = dyn_cast<PHINode>(X->getArgOperand(0));
106 assert(PN && "Expected Phi Node!");
108 // Don't create a new Phi unless we can remove the old one.
109 if (!PN->hasOneUse())
112 for (Value *IncValPhi : PN->incoming_values()) {
113 auto *Reinterpret = isReinterpretToSVBool(IncValPhi);
115 RequiredType != Reinterpret->getArgOperand(0)->getType())
119 // Create the new Phi
120 LLVMContext &Ctx = PN->getContext();
121 IRBuilder<> Builder(Ctx);
122 Builder.SetInsertPoint(PN);
123 PHINode *NPN = Builder.CreatePHI(RequiredType, PN->getNumIncomingValues());
124 Worklist.push_back(PN);
126 for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) {
127 auto *Reinterpret = cast<Instruction>(PN->getIncomingValue(I));
128 NPN->addIncoming(Reinterpret->getOperand(0), PN->getIncomingBlock(I));
129 Worklist.push_back(Reinterpret);
132 // Cleanup Phi Node and reinterprets
133 X->replaceAllUsesWith(NPN);
134 X->eraseFromParent();
136 for (auto &I : Worklist)
138 I->eraseFromParent();
143 bool SVEIntrinsicOpts::optimizePTest(IntrinsicInst *I) {
144 IntrinsicInst *Op1 = dyn_cast<IntrinsicInst>(I->getArgOperand(0));
145 IntrinsicInst *Op2 = dyn_cast<IntrinsicInst>(I->getArgOperand(1));
148 Op1->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
149 Op2->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
150 Op1->getArgOperand(0)->getType() == Op2->getArgOperand(0)->getType()) {
152 Value *Ops[] = {Op1->getArgOperand(0), Op2->getArgOperand(0)};
153 Type *Tys[] = {Op1->getArgOperand(0)->getType()};
154 Module *M = I->getParent()->getParent()->getParent();
156 auto Fn = Intrinsic::getDeclaration(M, I->getIntrinsicID(), Tys);
157 auto CI = CallInst::Create(Fn, Ops, I->getName(), I);
159 I->replaceAllUsesWith(CI);
160 I->eraseFromParent();
161 if (Op1->use_empty())
162 Op1->eraseFromParent();
163 if (Op1 != Op2 && Op2->use_empty())
164 Op2->eraseFromParent();
172 bool SVEIntrinsicOpts::optimizeConvertFromSVBool(IntrinsicInst *I) {
173 assert(I->getIntrinsicID() == Intrinsic::aarch64_sve_convert_from_svbool &&
174 "Unexpected opcode");
176 // If the reinterpret instruction operand is a PHI Node
177 if (isa<PHINode>(I->getArgOperand(0)))
178 return processPhiNode(I);
180 // If we have a reinterpret intrinsic I of type A which is converting from
181 // another reinterpret Y of type B, and the source type of Y is A, then we can
182 // elide away both reinterprets if there are no other users of Y.
183 auto *Y = isReinterpretToSVBool(I->getArgOperand(0));
187 Value *SourceVal = Y->getArgOperand(0);
188 if (I->getType() != SourceVal->getType())
191 I->replaceAllUsesWith(SourceVal);
192 I->eraseFromParent();
194 Y->eraseFromParent();
199 bool SVEIntrinsicOpts::optimizeIntrinsic(Instruction *I) {
200 IntrinsicInst *IntrI = dyn_cast<IntrinsicInst>(I);
204 switch (IntrI->getIntrinsicID()) {
205 case Intrinsic::aarch64_sve_convert_from_svbool:
206 return optimizeConvertFromSVBool(IntrI);
207 case Intrinsic::aarch64_sve_ptest_any:
208 case Intrinsic::aarch64_sve_ptest_first:
209 case Intrinsic::aarch64_sve_ptest_last:
210 return optimizePTest(IntrI);
218 bool SVEIntrinsicOpts::optimizeFunctions(
219 SmallSetVector<Function *, 4> &Functions) {
220 bool Changed = false;
221 for (auto *F : Functions) {
222 DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>(*F).getDomTree();
224 // Traverse the DT with an rpo walk so we see defs before uses, allowing
225 // simplification to be done incrementally.
226 BasicBlock *Root = DT->getRoot();
227 ReversePostOrderTraversal<BasicBlock *> RPOT(Root);
228 for (auto *BB : RPOT)
229 for (Instruction &I : make_early_inc_range(*BB))
230 Changed |= optimizeIntrinsic(&I);
235 bool SVEIntrinsicOpts::runOnModule(Module &M) {
236 bool Changed = false;
237 SmallSetVector<Function *, 4> Functions;
239 // Check for SVE intrinsic declarations first so that we only iterate over
240 // relevant functions. Where an appropriate declaration is found, store the
241 // function(s) where it is used so we can target these only.
242 for (auto &F : M.getFunctionList()) {
243 if (!F.isDeclaration())
246 switch (F.getIntrinsicID()) {
247 case Intrinsic::aarch64_sve_convert_from_svbool:
248 case Intrinsic::aarch64_sve_ptest_any:
249 case Intrinsic::aarch64_sve_ptest_first:
250 case Intrinsic::aarch64_sve_ptest_last:
251 for (auto I = F.user_begin(), E = F.user_end(); I != E;) {
252 auto *Inst = dyn_cast<Instruction>(*I++);
253 Functions.insert(Inst->getFunction());
261 if (!Functions.empty())
262 Changed |= optimizeFunctions(Functions);