1 //===--- AMDGPUPropagateAttributes.cpp --------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// \brief This pass propagates attributes from kernels to the non-entry
11 /// functions. Most of the library functions were not compiled for specific ABI,
12 /// yet will be correctly compiled if proper attrbutes are propagated from the
15 /// The pass analyzes call graph and propagates ABI target features through the
18 /// It can run in two modes: as a function or module pass. A function pass
19 /// simply propagates attributes. A module pass clones functions if there are
20 /// callers with different ABI. If a function is clonned all call sites will
21 /// be updated to use a correct clone.
23 /// A function pass is limited in functionality but can run early in the
24 /// pipeline. A module pass is more powerful but has to run late, so misses
25 /// library folding opportunities.
27 //===----------------------------------------------------------------------===//
30 #include "AMDGPUSubtarget.h"
31 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
32 #include "Utils/AMDGPUBaseInfo.h"
33 #include "llvm/ADT/SmallSet.h"
34 #include "llvm/ADT/SmallVector.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/Module.h"
37 #include "llvm/Target/TargetMachine.h"
38 #include "llvm/Transforms/Utils/Cloning.h"
41 #define DEBUG_TYPE "amdgpu-propagate-attributes"
46 extern const SubtargetFeatureKV AMDGPUFeatureKV[AMDGPU::NumSubtargetFeatures-1];
51 // Target features to propagate.
52 static constexpr const FeatureBitset TargetFeatures = {
53 AMDGPU::FeatureWavefrontSize16,
54 AMDGPU::FeatureWavefrontSize32,
55 AMDGPU::FeatureWavefrontSize64
58 // Attributes to propagate.
59 static constexpr const char* AttributeNames[] = {
63 static constexpr unsigned NumAttr =
64 sizeof(AttributeNames) / sizeof(AttributeNames[0]);
66 class AMDGPUPropagateAttributes {
70 explicit FnProperties(const FeatureBitset &&FB) : Features(FB) {}
73 explicit FnProperties(const TargetMachine &TM, const Function &F) {
74 Features = TM.getSubtargetImpl(F)->getFeatureBits();
76 for (unsigned I = 0; I < NumAttr; ++I)
77 if (F.hasFnAttribute(AttributeNames[I]))
78 Attributes[I] = F.getFnAttribute(AttributeNames[I]);
81 bool operator == (const FnProperties &Other) const {
82 if ((Features & TargetFeatures) != (Other.Features & TargetFeatures))
84 for (unsigned I = 0; I < NumAttr; ++I)
85 if (Attributes[I] != Other.Attributes[I])
90 FnProperties adjustToCaller(const FnProperties &CallerProps) const {
91 FnProperties New((Features & ~TargetFeatures) | CallerProps.Features);
92 for (unsigned I = 0; I < NumAttr; ++I)
93 New.Attributes[I] = CallerProps.Attributes[I];
97 FeatureBitset Features;
98 Optional<Attribute> Attributes[NumAttr];
103 Clone(const FnProperties &Props, Function *OrigF, Function *NewF) :
104 Properties(Props), OrigF(OrigF), NewF(NewF) {}
106 FnProperties Properties;
111 const TargetMachine *TM;
113 // Clone functions as needed or just set attributes.
116 // Option propagation roots.
117 SmallSet<Function *, 32> Roots;
119 // Clones of functions with their attributes.
120 SmallVector<Clone, 32> Clones;
122 // Find a clone with required features.
123 Function *findFunction(const FnProperties &PropsNeeded,
126 // Clone function \p F and set \p NewProps on the clone.
127 // Cole takes the name of original function.
128 Function *cloneWithProperties(Function &F, const FnProperties &NewProps);
130 // Set new function's features in place.
131 void setFeatures(Function &F, const FeatureBitset &NewFeatures);
133 // Set new function's attributes in place.
134 void setAttributes(Function &F, const ArrayRef<Optional<Attribute>> NewAttrs);
136 std::string getFeatureString(const FeatureBitset &Features) const;
138 // Propagate attributes from Roots.
142 AMDGPUPropagateAttributes(const TargetMachine *TM, bool AllowClone) :
143 TM(TM), AllowClone(AllowClone) {}
145 // Use F as a root and propagate its attributes.
146 bool process(Function &F);
148 // Propagate attributes starting from kernel functions.
149 bool process(Module &M);
152 // Allows to propagate attributes early, but no clonning is allowed as it must
153 // be a function pass to run before any optimizations.
154 // TODO: We shall only need a one instance of module pass, but that needs to be
155 // in the linker pipeline which is currently not possible.
156 class AMDGPUPropagateAttributesEarly : public FunctionPass {
157 const TargetMachine *TM;
160 static char ID; // Pass identification
162 AMDGPUPropagateAttributesEarly(const TargetMachine *TM = nullptr) :
163 FunctionPass(ID), TM(TM) {
164 initializeAMDGPUPropagateAttributesEarlyPass(
165 *PassRegistry::getPassRegistry());
168 bool runOnFunction(Function &F) override;
171 // Allows to propagate attributes with clonning but does that late in the
173 class AMDGPUPropagateAttributesLate : public ModulePass {
174 const TargetMachine *TM;
177 static char ID; // Pass identification
179 AMDGPUPropagateAttributesLate(const TargetMachine *TM = nullptr) :
180 ModulePass(ID), TM(TM) {
181 initializeAMDGPUPropagateAttributesLatePass(
182 *PassRegistry::getPassRegistry());
185 bool runOnModule(Module &M) override;
188 } // end anonymous namespace.
190 char AMDGPUPropagateAttributesEarly::ID = 0;
191 char AMDGPUPropagateAttributesLate::ID = 0;
193 INITIALIZE_PASS(AMDGPUPropagateAttributesEarly,
194 "amdgpu-propagate-attributes-early",
195 "Early propagate attributes from kernels to functions",
197 INITIALIZE_PASS(AMDGPUPropagateAttributesLate,
198 "amdgpu-propagate-attributes-late",
199 "Late propagate attributes from kernels to functions",
203 AMDGPUPropagateAttributes::findFunction(const FnProperties &PropsNeeded,
205 // TODO: search for clone's clones.
206 for (Clone &C : Clones)
207 if (C.OrigF == OrigF && PropsNeeded == C.Properties)
213 bool AMDGPUPropagateAttributes::process(Module &M) {
214 for (auto &F : M.functions())
215 if (AMDGPU::isEntryFunctionCC(F.getCallingConv()))
221 bool AMDGPUPropagateAttributes::process(Function &F) {
226 bool AMDGPUPropagateAttributes::process() {
227 bool Changed = false;
228 SmallSet<Function *, 32> NewRoots;
229 SmallSet<Function *, 32> Replaced;
233 Module &M = *(*Roots.begin())->getParent();
236 Roots.insert(NewRoots.begin(), NewRoots.end());
239 for (auto &F : M.functions()) {
240 if (F.isDeclaration())
243 const FnProperties CalleeProps(*TM, F);
244 SmallVector<std::pair<CallBase *, Function *>, 32> ToReplace;
245 SmallSet<CallBase *, 32> Visited;
247 for (User *U : F.users()) {
248 Instruction *I = dyn_cast<Instruction>(U);
251 CallBase *CI = dyn_cast<CallBase>(I);
254 Function *Caller = CI->getCaller();
255 if (!Caller || !Visited.insert(CI).second)
257 if (!Roots.count(Caller) && !NewRoots.count(Caller))
260 const FnProperties CallerProps(*TM, *Caller);
262 if (CalleeProps == CallerProps) {
263 if (!Roots.count(&F))
268 Function *NewF = findFunction(CallerProps, &F);
270 const FnProperties NewProps = CalleeProps.adjustToCaller(CallerProps);
272 // This may set different features on different iteartions if
273 // there is a contradiction in callers' attributes. In this case
274 // we rely on a second pass running on Module, which is allowed
276 setFeatures(F, NewProps.Features);
277 setAttributes(F, NewProps.Attributes);
283 NewF = cloneWithProperties(F, NewProps);
284 Clones.push_back(Clone(CallerProps, &F, NewF));
285 NewRoots.insert(NewF);
288 ToReplace.push_back(std::make_pair(CI, NewF));
294 while (!ToReplace.empty()) {
295 auto R = ToReplace.pop_back_val();
296 R.first->setCalledFunction(R.second);
299 } while (!NewRoots.empty());
301 for (Function *F : Replaced) {
303 F->eraseFromParent();
313 AMDGPUPropagateAttributes::cloneWithProperties(Function &F,
314 const FnProperties &NewProps) {
315 LLVM_DEBUG(dbgs() << "Cloning " << F.getName() << '\n');
317 ValueToValueMapTy dummy;
318 Function *NewF = CloneFunction(&F, dummy);
319 setFeatures(*NewF, NewProps.Features);
320 setAttributes(*NewF, NewProps.Attributes);
321 NewF->setVisibility(GlobalValue::DefaultVisibility);
322 NewF->setLinkage(GlobalValue::InternalLinkage);
324 // Swap names. If that is the only clone it will retain the name of now
325 // dead value. Preserve original name for externally visible functions.
326 if (F.hasName() && F.hasLocalLinkage()) {
327 std::string NewName = std::string(NewF->getName());
335 void AMDGPUPropagateAttributes::setFeatures(Function &F,
336 const FeatureBitset &NewFeatures) {
337 std::string NewFeatureStr = getFeatureString(NewFeatures);
339 LLVM_DEBUG(dbgs() << "Set features "
340 << getFeatureString(NewFeatures & TargetFeatures)
341 << " on " << F.getName() << '\n');
343 F.removeFnAttr("target-features");
344 F.addFnAttr("target-features", NewFeatureStr);
347 void AMDGPUPropagateAttributes::setAttributes(Function &F,
348 const ArrayRef<Optional<Attribute>> NewAttrs) {
349 LLVM_DEBUG(dbgs() << "Set attributes on " << F.getName() << ":\n");
350 for (unsigned I = 0; I < NumAttr; ++I) {
351 F.removeFnAttr(AttributeNames[I]);
353 LLVM_DEBUG(dbgs() << '\t' << NewAttrs[I]->getAsString() << '\n');
354 F.addFnAttr(*NewAttrs[I]);
360 AMDGPUPropagateAttributes::getFeatureString(const FeatureBitset &Features) const
363 for (const SubtargetFeatureKV &KV : AMDGPUFeatureKV) {
364 if (Features[KV.Value])
365 Ret += (StringRef("+") + KV.Key + ",").str();
366 else if (TargetFeatures[KV.Value])
367 Ret += (StringRef("-") + KV.Key + ",").str();
369 Ret.pop_back(); // Remove last comma.
373 bool AMDGPUPropagateAttributesEarly::runOnFunction(Function &F) {
374 if (!TM || !AMDGPU::isEntryFunctionCC(F.getCallingConv()))
377 return AMDGPUPropagateAttributes(TM, false).process(F);
380 bool AMDGPUPropagateAttributesLate::runOnModule(Module &M) {
384 return AMDGPUPropagateAttributes(TM, true).process(M);
388 *llvm::createAMDGPUPropagateAttributesEarlyPass(const TargetMachine *TM) {
389 return new AMDGPUPropagateAttributesEarly(TM);
393 *llvm::createAMDGPUPropagateAttributesLatePass(const TargetMachine *TM) {
394 return new AMDGPUPropagateAttributesLate(TM);