1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/DemandedBits.h"
17 #include "llvm/Analysis/GlobalsModRef.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/Analysis/VectorUtils.h"
20 #include "llvm/IR/InstIterator.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/Transforms/Utils.h"
23 #include "llvm/Transforms/Utils/ModuleUtils.h"
27 #define DEBUG_TYPE "inject-tli-mappings"
29 STATISTIC(NumCallInjected,
30 "Number of calls in which the mappings have been injected.");
32 STATISTIC(NumVFDeclAdded,
33 "Number of function declarations that have been added.");
34 STATISTIC(NumCompUsedAdded,
35 "Number of `@llvm.compiler.used` operands that have been added.");
37 /// A helper function that adds the vector function declaration that
38 /// vectorizes the CallInst CI with a vectorization factor of VF
39 /// lanes. The TLI assumes that all parameters and the return type of
40 /// CI (other than void) need to be widened to a VectorType of VF
42 static void addVariantDeclaration(CallInst &CI, const unsigned VF,
43 const StringRef VFName) {
44 Module *M = CI.getModule();
46 // Add function declaration.
47 Type *RetTy = ToVectorTy(CI.getType(), VF);
48 SmallVector<Type *, 4> Tys;
49 for (Value *ArgOperand : CI.arg_operands())
50 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
51 assert(!CI.getFunctionType()->isVarArg() &&
52 "VarArg functions are not supported.");
53 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
55 Function::Create(FTy, Function::ExternalLinkage, VFName, M);
56 VectorF->copyAttributesFrom(CI.getCalledFunction());
58 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
59 << "` of type " << *(VectorF->getType()) << "\n");
61 // Make function declaration (without a body) "sticky" in the IR by
62 // listing it in the @llvm.compiler.used intrinsic.
63 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
64 "only on declarations.");
65 appendToCompilerUsed(*M, {VectorF});
66 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
67 << "` to `@llvm.compiler.used`.\n");
71 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
72 // This is needed to make sure we don't query the TLI for calls to
73 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
74 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
75 // as such calls make the `isFunctionVectorizable` raise an
77 if (CI.isNoBuiltin() || !CI.getCalledFunction())
80 const std::string ScalarName = std::string(CI.getCalledFunction()->getName());
81 // Nothing to be done if the TLI thinks the function is not
83 if (!TLI.isFunctionVectorizable(ScalarName))
85 SmallVector<std::string, 8> Mappings;
86 VFABI::getVectorVariantNames(CI, Mappings);
87 Module *M = CI.getModule();
88 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
90 // All VFs in the TLI are powers of 2.
91 for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
93 const std::string TLIName =
94 std::string(TLI.getVectorizedFunction(ScalarName, VF));
95 if (!TLIName.empty()) {
96 std::string MangledName = VFABI::mangleTLIVectorName(
97 TLIName, ScalarName, CI.getNumArgOperands(), VF);
98 if (!OriginalSetOfMappings.count(MangledName)) {
99 Mappings.push_back(MangledName);
102 Function *VariantF = M->getFunction(TLIName);
104 addVariantDeclaration(CI, VF, TLIName);
108 VFABI::setVectorVariantNames(&CI, Mappings);
111 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
112 for (auto &I : instructions(F))
113 if (auto CI = dyn_cast<CallInst>(&I))
114 addMappingsFromTLI(TLI, *CI);
115 // Even if the pass adds IR attributes, the analyses are preserved.
119 ////////////////////////////////////////////////////////////////////////////////
120 // New pass manager implementation.
121 ////////////////////////////////////////////////////////////////////////////////
122 PreservedAnalyses InjectTLIMappings::run(Function &F,
123 FunctionAnalysisManager &AM) {
124 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
126 // Even if the pass adds IR attributes, the analyses are preserved.
127 return PreservedAnalyses::all();
130 ////////////////////////////////////////////////////////////////////////////////
131 // Legacy PM Implementation.
132 ////////////////////////////////////////////////////////////////////////////////
133 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
134 const TargetLibraryInfo &TLI =
135 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
136 return runImpl(TLI, F);
139 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
140 AU.setPreservesCFG();
141 AU.addRequired<TargetLibraryInfoWrapperPass>();
142 AU.addPreserved<TargetLibraryInfoWrapperPass>();
143 AU.addPreserved<ScalarEvolutionWrapperPass>();
144 AU.addPreserved<AAResultsWrapperPass>();
145 AU.addPreserved<LoopAccessLegacyAnalysis>();
146 AU.addPreserved<DemandedBitsWrapperPass>();
147 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
148 AU.addPreserved<GlobalsAAWrapperPass>();
151 ////////////////////////////////////////////////////////////////////////////////
152 // Legacy Pass manager initialization
153 ////////////////////////////////////////////////////////////////////////////////
154 char InjectTLIMappingsLegacy::ID = 0;
156 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
157 "Inject TLI Mappings", false, false)
158 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
159 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
162 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
163 return new InjectTLIMappingsLegacy();