1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 #include "CGLoopInfo.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Attr.h"
13 #include "llvm/IR/BasicBlock.h"
14 #include "llvm/IR/CFG.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/InstrTypes.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/Metadata.h"
19 using namespace clang::CodeGen;
22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs,
23 const llvm::DebugLoc &StartLoc,
24 const llvm::DebugLoc &EndLoc, MDNode *&AccGroup) {
26 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
27 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
28 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
29 Attrs.PipelineInitiationInterval == 0 &&
30 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
31 Attrs.UnrollEnable == LoopAttributes::Unspecified &&
32 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
33 Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
37 SmallVector<Metadata *, 4> Args;
38 // Reserve operand 0 for loop id self reference.
39 auto TempNode = MDNode::getTemporary(Ctx, None);
40 Args.push_back(TempNode.get());
42 // If we have a valid start debug location for the loop, add it.
44 Args.push_back(StartLoc.getAsMDNode());
46 // If we also have a valid end debug location for the loop, add it.
48 Args.push_back(EndLoc.getAsMDNode());
51 // Setting vectorize.width
52 if (Attrs.VectorizeWidth > 0) {
53 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"),
54 ConstantAsMetadata::get(ConstantInt::get(
55 Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))};
56 Args.push_back(MDNode::get(Ctx, Vals));
59 // Setting interleave.count
60 if (Attrs.InterleaveCount > 0) {
61 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"),
62 ConstantAsMetadata::get(ConstantInt::get(
63 Type::getInt32Ty(Ctx), Attrs.InterleaveCount))};
64 Args.push_back(MDNode::get(Ctx, Vals));
67 // Setting unroll.count
68 if (Attrs.UnrollCount > 0) {
69 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
70 ConstantAsMetadata::get(ConstantInt::get(
71 Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
72 Args.push_back(MDNode::get(Ctx, Vals));
75 // Setting unroll_and_jam.count
76 if (Attrs.UnrollAndJamCount > 0) {
77 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
78 ConstantAsMetadata::get(ConstantInt::get(
79 Type::getInt32Ty(Ctx), Attrs.UnrollAndJamCount))};
80 Args.push_back(MDNode::get(Ctx, Vals));
83 // Setting vectorize.enable
84 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
85 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
86 ConstantAsMetadata::get(ConstantInt::get(
87 Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable ==
88 LoopAttributes::Enable)))};
89 Args.push_back(MDNode::get(Ctx, Vals));
92 // Setting unroll.full or unroll.disable
93 if (Attrs.UnrollEnable != LoopAttributes::Unspecified) {
95 if (Attrs.UnrollEnable == LoopAttributes::Enable)
96 Name = "llvm.loop.unroll.enable";
97 else if (Attrs.UnrollEnable == LoopAttributes::Full)
98 Name = "llvm.loop.unroll.full";
100 Name = "llvm.loop.unroll.disable";
101 Metadata *Vals[] = {MDString::get(Ctx, Name)};
102 Args.push_back(MDNode::get(Ctx, Vals));
105 // Setting unroll_and_jam.full or unroll_and_jam.disable
106 if (Attrs.UnrollAndJamEnable != LoopAttributes::Unspecified) {
108 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable)
109 Name = "llvm.loop.unroll_and_jam.enable";
110 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Full)
111 Name = "llvm.loop.unroll_and_jam.full";
113 Name = "llvm.loop.unroll_and_jam.disable";
114 Metadata *Vals[] = {MDString::get(Ctx, Name)};
115 Args.push_back(MDNode::get(Ctx, Vals));
118 if (Attrs.DistributeEnable != LoopAttributes::Unspecified) {
119 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
120 ConstantAsMetadata::get(ConstantInt::get(
121 Type::getInt1Ty(Ctx), (Attrs.DistributeEnable ==
122 LoopAttributes::Enable)))};
123 Args.push_back(MDNode::get(Ctx, Vals));
126 if (Attrs.IsParallel) {
127 AccGroup = MDNode::getDistinct(Ctx, {});
128 Args.push_back(MDNode::get(
129 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
132 if (Attrs.PipelineDisabled) {
134 MDString::get(Ctx, "llvm.loop.pipeline.disable"),
135 ConstantAsMetadata::get(ConstantInt::get(
136 Type::getInt1Ty(Ctx), (Attrs.PipelineDisabled == true)))};
137 Args.push_back(MDNode::get(Ctx, Vals));
140 if (Attrs.PipelineInitiationInterval > 0) {
142 MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
143 ConstantAsMetadata::get(ConstantInt::get(
144 Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
145 Args.push_back(MDNode::get(Ctx, Vals));
148 // Set the first operand to itself.
149 MDNode *LoopID = MDNode::get(Ctx, Args);
150 LoopID->replaceOperandWith(0, LoopID);
154 LoopAttributes::LoopAttributes(bool IsParallel)
155 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
156 UnrollEnable(LoopAttributes::Unspecified),
157 UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
158 InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
159 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
160 PipelineInitiationInterval(0) {}
162 void LoopAttributes::clear() {
167 UnrollAndJamCount = 0;
168 VectorizeEnable = LoopAttributes::Unspecified;
169 UnrollEnable = LoopAttributes::Unspecified;
170 UnrollAndJamEnable = LoopAttributes::Unspecified;
171 DistributeEnable = LoopAttributes::Unspecified;
172 PipelineDisabled = false;
173 PipelineInitiationInterval = 0;
176 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
177 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
178 : LoopID(nullptr), Header(Header), Attrs(Attrs) {
180 createMetadata(Header->getContext(), Attrs, StartLoc, EndLoc, AccGroup);
183 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
184 const llvm::DebugLoc &EndLoc) {
185 Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc));
186 // Clear the attributes so nested loops do not inherit them.
190 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
191 ArrayRef<const clang::Attr *> Attrs,
192 const llvm::DebugLoc &StartLoc,
193 const llvm::DebugLoc &EndLoc) {
195 // Identify loop hint attributes from Attrs.
196 for (const auto *Attr : Attrs) {
197 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
198 const OpenCLUnrollHintAttr *OpenCLHint =
199 dyn_cast<OpenCLUnrollHintAttr>(Attr);
201 // Skip non loop hint attributes
202 if (!LH && !OpenCLHint) {
206 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
207 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
208 unsigned ValueInt = 1;
209 // Translate opencl_unroll_hint attribute argument to
210 // equivalent LoopHintAttr enums.
211 // OpenCL v2.0 s6.11.5:
212 // 0 - full unroll (no argument).
213 // 1 - disable unroll.
214 // other positive integer n - unroll by n.
216 ValueInt = OpenCLHint->getUnrollHint();
218 State = LoopHintAttr::Full;
219 } else if (ValueInt != 1) {
220 Option = LoopHintAttr::UnrollCount;
221 State = LoopHintAttr::Numeric;
224 auto *ValueExpr = LH->getValue();
226 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
227 ValueInt = ValueAPS.getSExtValue();
230 Option = LH->getOption();
231 State = LH->getState();
234 case LoopHintAttr::Disable:
236 case LoopHintAttr::Vectorize:
237 // Disable vectorization by specifying a width of 1.
238 setVectorizeWidth(1);
240 case LoopHintAttr::Interleave:
241 // Disable interleaving by speciyfing a count of 1.
242 setInterleaveCount(1);
244 case LoopHintAttr::Unroll:
245 setUnrollState(LoopAttributes::Disable);
247 case LoopHintAttr::UnrollAndJam:
248 setUnrollAndJamState(LoopAttributes::Disable);
250 case LoopHintAttr::Distribute:
251 setDistributeState(false);
253 case LoopHintAttr::PipelineDisabled:
254 setPipelineDisabled(true);
256 case LoopHintAttr::UnrollCount:
257 case LoopHintAttr::UnrollAndJamCount:
258 case LoopHintAttr::VectorizeWidth:
259 case LoopHintAttr::InterleaveCount:
260 case LoopHintAttr::PipelineInitiationInterval:
261 llvm_unreachable("Options cannot be disabled.");
265 case LoopHintAttr::Enable:
267 case LoopHintAttr::Vectorize:
268 case LoopHintAttr::Interleave:
269 setVectorizeEnable(true);
271 case LoopHintAttr::Unroll:
272 setUnrollState(LoopAttributes::Enable);
274 case LoopHintAttr::UnrollAndJam:
275 setUnrollAndJamState(LoopAttributes::Enable);
277 case LoopHintAttr::Distribute:
278 setDistributeState(true);
280 case LoopHintAttr::UnrollCount:
281 case LoopHintAttr::UnrollAndJamCount:
282 case LoopHintAttr::VectorizeWidth:
283 case LoopHintAttr::InterleaveCount:
284 case LoopHintAttr::PipelineDisabled:
285 case LoopHintAttr::PipelineInitiationInterval:
286 llvm_unreachable("Options cannot enabled.");
290 case LoopHintAttr::AssumeSafety:
292 case LoopHintAttr::Vectorize:
293 case LoopHintAttr::Interleave:
294 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
296 setVectorizeEnable(true);
298 case LoopHintAttr::Unroll:
299 case LoopHintAttr::UnrollAndJam:
300 case LoopHintAttr::UnrollCount:
301 case LoopHintAttr::UnrollAndJamCount:
302 case LoopHintAttr::VectorizeWidth:
303 case LoopHintAttr::InterleaveCount:
304 case LoopHintAttr::Distribute:
305 case LoopHintAttr::PipelineDisabled:
306 case LoopHintAttr::PipelineInitiationInterval:
307 llvm_unreachable("Options cannot be used to assume mem safety.");
311 case LoopHintAttr::Full:
313 case LoopHintAttr::Unroll:
314 setUnrollState(LoopAttributes::Full);
316 case LoopHintAttr::UnrollAndJam:
317 setUnrollAndJamState(LoopAttributes::Full);
319 case LoopHintAttr::Vectorize:
320 case LoopHintAttr::Interleave:
321 case LoopHintAttr::UnrollCount:
322 case LoopHintAttr::UnrollAndJamCount:
323 case LoopHintAttr::VectorizeWidth:
324 case LoopHintAttr::InterleaveCount:
325 case LoopHintAttr::Distribute:
326 case LoopHintAttr::PipelineDisabled:
327 case LoopHintAttr::PipelineInitiationInterval:
328 llvm_unreachable("Options cannot be used with 'full' hint.");
332 case LoopHintAttr::Numeric:
334 case LoopHintAttr::VectorizeWidth:
335 setVectorizeWidth(ValueInt);
337 case LoopHintAttr::InterleaveCount:
338 setInterleaveCount(ValueInt);
340 case LoopHintAttr::UnrollCount:
341 setUnrollCount(ValueInt);
343 case LoopHintAttr::UnrollAndJamCount:
344 setUnrollAndJamCount(ValueInt);
346 case LoopHintAttr::PipelineInitiationInterval:
347 setPipelineInitiationInterval(ValueInt);
349 case LoopHintAttr::Unroll:
350 case LoopHintAttr::UnrollAndJam:
351 case LoopHintAttr::Vectorize:
352 case LoopHintAttr::Interleave:
353 case LoopHintAttr::Distribute:
354 case LoopHintAttr::PipelineDisabled:
355 llvm_unreachable("Options cannot be assigned a value.");
362 /// Stage the attributes.
363 push(Header, StartLoc, EndLoc);
366 void LoopInfoStack::pop() {
367 assert(!Active.empty() && "No active loops to pop");
371 void LoopInfoStack::InsertHelper(Instruction *I) const {
372 if (I->mayReadOrWriteMemory()) {
373 SmallVector<Metadata *, 4> AccessGroups;
374 for (const LoopInfo &AL : Active) {
375 // Here we assume that every loop that has an access group is parallel.
376 if (MDNode *Group = AL.getAccessGroup())
377 AccessGroups.push_back(Group);
379 MDNode *UnionMD = nullptr;
380 if (AccessGroups.size() == 1)
381 UnionMD = cast<MDNode>(AccessGroups[0]);
382 else if (AccessGroups.size() >= 2)
383 UnionMD = MDNode::get(I->getContext(), AccessGroups);
384 I->setMetadata("llvm.access.group", UnionMD);
390 const LoopInfo &L = getInfo();
394 if (I->isTerminator()) {
395 for (BasicBlock *Succ : successors(I))
396 if (Succ == L.getHeader()) {
397 I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());