contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp

   1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "CGLoopInfo.h"
  10 #include "clang/AST/ASTContext.h"
  11 #include "clang/AST/Attr.h"
  12 #include "llvm/IR/BasicBlock.h"
  13 #include "llvm/IR/CFG.h"
  14 #include "llvm/IR/Constants.h"
  15 #include "llvm/IR/InstrTypes.h"
  16 #include "llvm/IR/Instructions.h"
  17 #include "llvm/IR/Metadata.h"
  18 using namespace clang::CodeGen;
  19 using namespace llvm;
  20
  21 MDNode *
  22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
  23   LLVMContext &Ctx = Header->getContext();
  24   SmallVector<Metadata *, 4> NewLoopProperties;
  25   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
  26   NewLoopProperties.push_back(TempNode.get());
  27   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
  28
  29   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
  30   LoopID->replaceOperandWith(0, LoopID);
  31   return LoopID;
  32 }
  33
  34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
  35                                            ArrayRef<Metadata *> LoopProperties,
  36                                            bool &HasUserTransforms) {
  37   LLVMContext &Ctx = Header->getContext();
  38
  39   Optional<bool> Enabled;
  40   if (Attrs.PipelineDisabled)
  41     Enabled = false;
  42   else if (Attrs.PipelineInitiationInterval != 0)
  43     Enabled = true;
  44
  45   if (Enabled != true) {
  46     SmallVector<Metadata *, 4> NewLoopProperties;
  47     if (Enabled == false) {
  48       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
  49       NewLoopProperties.push_back(
  50           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
  51                             ConstantAsMetadata::get(ConstantInt::get(
  52                                 llvm::Type::getInt1Ty(Ctx), 1))}));
  53       LoopProperties = NewLoopProperties;
  54     }
  55     return createLoopPropertiesMetadata(LoopProperties);
  56   }
  57
  58   SmallVector<Metadata *, 4> Args;
  59   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
  60   Args.push_back(TempNode.get());
  61   Args.append(LoopProperties.begin(), LoopProperties.end());
  62
  63   if (Attrs.PipelineInitiationInterval > 0) {
  64     Metadata *Vals[] = {
  65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
  66         ConstantAsMetadata::get(ConstantInt::get(
  67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
  68     Args.push_back(MDNode::get(Ctx, Vals));
  69   }
  70
  71   // No follow-up: This is the last transformation.
  72
  73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
  74   LoopID->replaceOperandWith(0, LoopID);
  75   HasUserTransforms = true;
  76   return LoopID;
  77 }
  78
  79 MDNode *
  80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
  81                                       ArrayRef<Metadata *> LoopProperties,
  82                                       bool &HasUserTransforms) {
  83   LLVMContext &Ctx = Header->getContext();
  84
  85   Optional<bool> Enabled;
  86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
  87     Enabled = false;
  88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
  89     Enabled = None;
  90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
  91            Attrs.UnrollCount != 0)
  92     Enabled = true;
  93
  94   if (Enabled != true) {
  95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
  96     // if unrolling is disabled.
  97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
  98   }
  99
 100   SmallVector<Metadata *, 4> FollowupLoopProperties;
 101
 102   // Apply all loop properties to the unrolled loop.
 103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 104
 105   // Don't unroll an already unrolled loop.
 106   FollowupLoopProperties.push_back(
 107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
 108
 109   bool FollowupHasTransforms = false;
 110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
 111                                               FollowupHasTransforms);
 112
 113   SmallVector<Metadata *, 4> Args;
 114   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 115   Args.push_back(TempNode.get());
 116   Args.append(LoopProperties.begin(), LoopProperties.end());
 117
 118   // Setting unroll.count
 119   if (Attrs.UnrollCount > 0) {
 120     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
 121                         ConstantAsMetadata::get(ConstantInt::get(
 122                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
 123     Args.push_back(MDNode::get(Ctx, Vals));
 124   }
 125
 126   // Setting unroll.full or unroll.disable
 127   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
 128     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
 129     Args.push_back(MDNode::get(Ctx, Vals));
 130   }
 131
 132   if (FollowupHasTransforms)
 133     Args.push_back(MDNode::get(
 134         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
 135
 136   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
 137   LoopID->replaceOperandWith(0, LoopID);
 138   HasUserTransforms = true;
 139   return LoopID;
 140 }
 141
 142 MDNode *
 143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
 144                                      ArrayRef<Metadata *> LoopProperties,
 145                                      bool &HasUserTransforms) {
 146   LLVMContext &Ctx = Header->getContext();
 147
 148   Optional<bool> Enabled;
 149   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
 150     Enabled = false;
 151   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
 152            Attrs.UnrollAndJamCount != 0)
 153     Enabled = true;
 154
 155   if (Enabled != true) {
 156     SmallVector<Metadata *, 4> NewLoopProperties;
 157     if (Enabled == false) {
 158       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 159       NewLoopProperties.push_back(MDNode::get(
 160           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
 161       LoopProperties = NewLoopProperties;
 162     }
 163     return createPartialUnrollMetadata(Attrs, LoopProperties,
 164                                        HasUserTransforms);
 165   }
 166
 167   SmallVector<Metadata *, 4> FollowupLoopProperties;
 168   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 169   FollowupLoopProperties.push_back(
 170       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
 171
 172   bool FollowupHasTransforms = false;
 173   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
 174                                                  FollowupHasTransforms);
 175
 176   SmallVector<Metadata *, 4> Args;
 177   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 178   Args.push_back(TempNode.get());
 179   Args.append(LoopProperties.begin(), LoopProperties.end());
 180
 181   // Setting unroll_and_jam.count
 182   if (Attrs.UnrollAndJamCount > 0) {
 183     Metadata *Vals[] = {
 184         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
 185         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
 186                                                  Attrs.UnrollAndJamCount))};
 187     Args.push_back(MDNode::get(Ctx, Vals));
 188   }
 189
 190   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
 191     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
 192     Args.push_back(MDNode::get(Ctx, Vals));
 193   }
 194
 195   if (FollowupHasTransforms)
 196     Args.push_back(MDNode::get(
 197         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
 198               Followup}));
 199
 200   if (UnrollAndJamInnerFollowup)
 201     Args.push_back(MDNode::get(
 202         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
 203               UnrollAndJamInnerFollowup}));
 204
 205   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
 206   LoopID->replaceOperandWith(0, LoopID);
 207   HasUserTransforms = true;
 208   return LoopID;
 209 }
 210
 211 MDNode *
 212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
 213                                       ArrayRef<Metadata *> LoopProperties,
 214                                       bool &HasUserTransforms) {
 215   LLVMContext &Ctx = Header->getContext();
 216
 217   Optional<bool> Enabled;
 218   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
 219     Enabled = false;
 220   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
 221            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
 222            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
 223     Enabled = true;
 224
 225   if (Enabled != true) {
 226     SmallVector<Metadata *, 4> NewLoopProperties;
 227     if (Enabled == false) {
 228       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 229       NewLoopProperties.push_back(
 230           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
 231                             ConstantAsMetadata::get(ConstantInt::get(
 232                                 llvm::Type::getInt1Ty(Ctx), 0))}));
 233       LoopProperties = NewLoopProperties;
 234     }
 235     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
 236   }
 237
 238   // Apply all loop properties to the vectorized loop.
 239   SmallVector<Metadata *, 4> FollowupLoopProperties;
 240   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 241
 242   // Don't vectorize an already vectorized loop.
 243   FollowupLoopProperties.push_back(
 244       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
 245
 246   bool FollowupHasTransforms = false;
 247   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
 248                                                 FollowupHasTransforms);
 249
 250   SmallVector<Metadata *, 4> Args;
 251   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 252   Args.push_back(TempNode.get());
 253   Args.append(LoopProperties.begin(), LoopProperties.end());
 254
 255   // Setting vectorize.predicate
 256   bool IsVectorPredicateEnabled = false;
 257   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
 258       Attrs.VectorizeEnable != LoopAttributes::Disable &&
 259       Attrs.VectorizeWidth < 1) {
 260
 261     IsVectorPredicateEnabled =
 262         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
 263
 264     Metadata *Vals[] = {
 265         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
 266         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
 267                                                  IsVectorPredicateEnabled))};
 268     Args.push_back(MDNode::get(Ctx, Vals));
 269   }
 270
 271   // Setting vectorize.width
 272   if (Attrs.VectorizeWidth > 0) {
 273     // This implies vectorize.enable = true, but only add it when it is not
 274     // already enabled.
 275     if (Attrs.VectorizeEnable != LoopAttributes::Enable)
 276       Args.push_back(
 277           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
 278                             ConstantAsMetadata::get(ConstantInt::get(
 279                                 llvm::Type::getInt1Ty(Ctx), 1))}));
 280
 281     Metadata *Vals[] = {
 282         MDString::get(Ctx, "llvm.loop.vectorize.width"),
 283         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
 284                                                  Attrs.VectorizeWidth))};
 285     Args.push_back(MDNode::get(Ctx, Vals));
 286   }
 287
 288   // Setting interleave.count
 289   if (Attrs.InterleaveCount > 0) {
 290     Metadata *Vals[] = {
 291         MDString::get(Ctx, "llvm.loop.interleave.count"),
 292         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
 293                                                  Attrs.InterleaveCount))};
 294     Args.push_back(MDNode::get(Ctx, Vals));
 295   }
 296
 297   // Setting vectorize.enable
 298   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
 299       IsVectorPredicateEnabled) {
 300     Metadata *Vals[] = {
 301         MDString::get(Ctx, "llvm.loop.vectorize.enable"),
 302         ConstantAsMetadata::get(ConstantInt::get(
 303             llvm::Type::getInt1Ty(Ctx),
 304             IsVectorPredicateEnabled
 305                 ? true
 306                 : (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
 307     Args.push_back(MDNode::get(Ctx, Vals));
 308   }
 309
 310   if (FollowupHasTransforms)
 311     Args.push_back(MDNode::get(
 312         Ctx,
 313         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
 314
 315   MDNode *LoopID = MDNode::get(Ctx, Args);
 316   LoopID->replaceOperandWith(0, LoopID);
 317   HasUserTransforms = true;
 318   return LoopID;
 319 }
 320
 321 MDNode *
 322 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
 323                                        ArrayRef<Metadata *> LoopProperties,
 324                                        bool &HasUserTransforms) {
 325   LLVMContext &Ctx = Header->getContext();
 326
 327   Optional<bool> Enabled;
 328   if (Attrs.DistributeEnable == LoopAttributes::Disable)
 329     Enabled = false;
 330   if (Attrs.DistributeEnable == LoopAttributes::Enable)
 331     Enabled = true;
 332
 333   if (Enabled != true) {
 334     SmallVector<Metadata *, 4> NewLoopProperties;
 335     if (Enabled == false) {
 336       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 337       NewLoopProperties.push_back(
 338           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
 339                             ConstantAsMetadata::get(ConstantInt::get(
 340                                 llvm::Type::getInt1Ty(Ctx), 0))}));
 341       LoopProperties = NewLoopProperties;
 342     }
 343     return createLoopVectorizeMetadata(Attrs, LoopProperties,
 344                                        HasUserTransforms);
 345   }
 346
 347   bool FollowupHasTransforms = false;
 348   MDNode *Followup =
 349       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
 350
 351   SmallVector<Metadata *, 4> Args;
 352   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 353   Args.push_back(TempNode.get());
 354   Args.append(LoopProperties.begin(), LoopProperties.end());
 355
 356   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
 357                       ConstantAsMetadata::get(ConstantInt::get(
 358                           llvm::Type::getInt1Ty(Ctx),
 359                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
 360   Args.push_back(MDNode::get(Ctx, Vals));
 361
 362   if (FollowupHasTransforms)
 363     Args.push_back(MDNode::get(
 364         Ctx,
 365         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
 366
 367   MDNode *LoopID = MDNode::get(Ctx, Args);
 368   LoopID->replaceOperandWith(0, LoopID);
 369   HasUserTransforms = true;
 370   return LoopID;
 371 }
 372
 373 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
 374                                            ArrayRef<Metadata *> LoopProperties,
 375                                            bool &HasUserTransforms) {
 376   LLVMContext &Ctx = Header->getContext();
 377
 378   Optional<bool> Enabled;
 379   if (Attrs.UnrollEnable == LoopAttributes::Disable)
 380     Enabled = false;
 381   else if (Attrs.UnrollEnable == LoopAttributes::Full)
 382     Enabled = true;
 383
 384   if (Enabled != true) {
 385     SmallVector<Metadata *, 4> NewLoopProperties;
 386     if (Enabled == false) {
 387       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 388       NewLoopProperties.push_back(
 389           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
 390       LoopProperties = NewLoopProperties;
 391     }
 392     return createLoopDistributeMetadata(Attrs, LoopProperties,
 393                                         HasUserTransforms);
 394   }
 395
 396   SmallVector<Metadata *, 4> Args;
 397   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 398   Args.push_back(TempNode.get());
 399   Args.append(LoopProperties.begin(), LoopProperties.end());
 400   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
 401
 402   // No follow-up: there is no loop after full unrolling.
 403   // TODO: Warn if there are transformations after full unrolling.
 404
 405   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
 406   LoopID->replaceOperandWith(0, LoopID);
 407   HasUserTransforms = true;
 408   return LoopID;
 409 }
 410
 411 MDNode *LoopInfo::createMetadata(
 412     const LoopAttributes &Attrs,
 413     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
 414     bool &HasUserTransforms) {
 415   SmallVector<Metadata *, 3> LoopProperties;
 416
 417   // If we have a valid start debug location for the loop, add it.
 418   if (StartLoc) {
 419     LoopProperties.push_back(StartLoc.getAsMDNode());
 420
 421     // If we also have a valid end debug location for the loop, add it.
 422     if (EndLoc)
 423       LoopProperties.push_back(EndLoc.getAsMDNode());
 424   }
 425
 426   assert(!!AccGroup == Attrs.IsParallel &&
 427          "There must be an access group iff the loop is parallel");
 428   if (Attrs.IsParallel) {
 429     LLVMContext &Ctx = Header->getContext();
 430     LoopProperties.push_back(MDNode::get(
 431         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
 432   }
 433
 434   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
 435                         AdditionalLoopProperties.end());
 436   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
 437 }
 438
 439 LoopAttributes::LoopAttributes(bool IsParallel)
 440     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
 441       UnrollEnable(LoopAttributes::Unspecified),
 442       UnrollAndJamEnable(LoopAttributes::Unspecified),
 443       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
 444       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
 445       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
 446       PipelineInitiationInterval(0) {}
 447
 448 void LoopAttributes::clear() {
 449   IsParallel = false;
 450   VectorizeWidth = 0;
 451   InterleaveCount = 0;
 452   UnrollCount = 0;
 453   UnrollAndJamCount = 0;
 454   VectorizeEnable = LoopAttributes::Unspecified;
 455   UnrollEnable = LoopAttributes::Unspecified;
 456   UnrollAndJamEnable = LoopAttributes::Unspecified;
 457   VectorizePredicateEnable = LoopAttributes::Unspecified;
 458   DistributeEnable = LoopAttributes::Unspecified;
 459   PipelineDisabled = false;
 460   PipelineInitiationInterval = 0;
 461 }
 462
 463 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
 464                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
 465                    LoopInfo *Parent)
 466     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
 467       Parent(Parent) {
 468
 469   if (Attrs.IsParallel) {
 470     // Create an access group for this loop.
 471     LLVMContext &Ctx = Header->getContext();
 472     AccGroup = MDNode::getDistinct(Ctx, {});
 473   }
 474
 475   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
 476       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
 477       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
 478       Attrs.PipelineInitiationInterval == 0 &&
 479       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
 480       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
 481       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
 482       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
 483       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
 484       !EndLoc)
 485     return;
 486
 487   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
 488 }
 489
 490 void LoopInfo::finish() {
 491   // We did not annotate the loop body instructions because there are no
 492   // attributes for this loop.
 493   if (!TempLoopID)
 494     return;
 495
 496   MDNode *LoopID;
 497   LoopAttributes CurLoopAttr = Attrs;
 498   LLVMContext &Ctx = Header->getContext();
 499
 500   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
 501                  Parent->Attrs.UnrollAndJamCount != 0)) {
 502     // Parent unroll-and-jams this loop.
 503     // Split the transformations in those that happens before the unroll-and-jam
 504     // and those after.
 505
 506     LoopAttributes BeforeJam, AfterJam;
 507
 508     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
 509
 510     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
 511     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
 512     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
 513     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
 514     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
 515
 516     switch (Attrs.UnrollEnable) {
 517     case LoopAttributes::Unspecified:
 518     case LoopAttributes::Disable:
 519       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
 520       AfterJam.UnrollEnable = Attrs.UnrollEnable;
 521       break;
 522     case LoopAttributes::Full:
 523       BeforeJam.UnrollEnable = LoopAttributes::Full;
 524       break;
 525     case LoopAttributes::Enable:
 526       AfterJam.UnrollEnable = LoopAttributes::Enable;
 527       break;
 528     }
 529
 530     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
 531     AfterJam.UnrollCount = Attrs.UnrollCount;
 532     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
 533     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
 534
 535     // If this loop is subject of an unroll-and-jam by the parent loop, and has
 536     // an unroll-and-jam annotation itself, we have to decide whether to first
 537     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
 538     // UnrollAndJam pass processes loops from inner to outer, so we apply the
 539     // inner first.
 540     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
 541     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
 542
 543     // Set the inner followup metadata to process by the outer loop. Only
 544     // consider the first inner loop.
 545     if (!Parent->UnrollAndJamInnerFollowup) {
 546       // Splitting the attributes into a BeforeJam and an AfterJam part will
 547       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
 548       // to be forwarded to the AfterJam part. We detect the situation here and
 549       // add it manually.
 550       SmallVector<Metadata *, 1> BeforeLoopProperties;
 551       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
 552           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
 553           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
 554         BeforeLoopProperties.push_back(
 555             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
 556
 557       bool InnerFollowupHasTransform = false;
 558       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
 559                                              InnerFollowupHasTransform);
 560       if (InnerFollowupHasTransform)
 561         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
 562     }
 563
 564     CurLoopAttr = BeforeJam;
 565   }
 566
 567   bool HasUserTransforms = false;
 568   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
 569   TempLoopID->replaceAllUsesWith(LoopID);
 570 }
 571
 572 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
 573                          const llvm::DebugLoc &EndLoc) {
 574   Active.emplace_back(
 575       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
 576                    Active.empty() ? nullptr : Active.back().get()));
 577   // Clear the attributes so nested loops do not inherit them.
 578   StagedAttrs.clear();
 579 }
 580
 581 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
 582                          ArrayRef<const clang::Attr *> Attrs,
 583                          const llvm::DebugLoc &StartLoc,
 584                          const llvm::DebugLoc &EndLoc) {
 585
 586   // Identify loop hint attributes from Attrs.
 587   for (const auto *Attr : Attrs) {
 588     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
 589     const OpenCLUnrollHintAttr *OpenCLHint =
 590         dyn_cast<OpenCLUnrollHintAttr>(Attr);
 591
 592     // Skip non loop hint attributes
 593     if (!LH && !OpenCLHint) {
 594       continue;
 595     }
 596
 597     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
 598     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
 599     unsigned ValueInt = 1;
 600     // Translate opencl_unroll_hint attribute argument to
 601     // equivalent LoopHintAttr enums.
 602     // OpenCL v2.0 s6.11.5:
 603     // 0 - enable unroll (no argument).
 604     // 1 - disable unroll.
 605     // other positive integer n - unroll by n.
 606     if (OpenCLHint) {
 607       ValueInt = OpenCLHint->getUnrollHint();
 608       if (ValueInt == 0) {
 609         State = LoopHintAttr::Enable;
 610       } else if (ValueInt != 1) {
 611         Option = LoopHintAttr::UnrollCount;
 612         State = LoopHintAttr::Numeric;
 613       }
 614     } else if (LH) {
 615       auto *ValueExpr = LH->getValue();
 616       if (ValueExpr) {
 617         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
 618         ValueInt = ValueAPS.getSExtValue();
 619       }
 620
 621       Option = LH->getOption();
 622       State = LH->getState();
 623     }
 624     switch (State) {
 625     case LoopHintAttr::Disable:
 626       switch (Option) {
 627       case LoopHintAttr::Vectorize:
 628         // Disable vectorization by specifying a width of 1.
 629         setVectorizeWidth(1);
 630         break;
 631       case LoopHintAttr::Interleave:
 632         // Disable interleaving by speciyfing a count of 1.
 633         setInterleaveCount(1);
 634         break;
 635       case LoopHintAttr::Unroll:
 636         setUnrollState(LoopAttributes::Disable);
 637         break;
 638       case LoopHintAttr::UnrollAndJam:
 639         setUnrollAndJamState(LoopAttributes::Disable);
 640         break;
 641       case LoopHintAttr::VectorizePredicate:
 642         setVectorizePredicateState(LoopAttributes::Disable);
 643         break;
 644       case LoopHintAttr::Distribute:
 645         setDistributeState(false);
 646         break;
 647       case LoopHintAttr::PipelineDisabled:
 648         setPipelineDisabled(true);
 649         break;
 650       case LoopHintAttr::UnrollCount:
 651       case LoopHintAttr::UnrollAndJamCount:
 652       case LoopHintAttr::VectorizeWidth:
 653       case LoopHintAttr::InterleaveCount:
 654       case LoopHintAttr::PipelineInitiationInterval:
 655         llvm_unreachable("Options cannot be disabled.");
 656         break;
 657       }
 658       break;
 659     case LoopHintAttr::Enable:
 660       switch (Option) {
 661       case LoopHintAttr::Vectorize:
 662       case LoopHintAttr::Interleave:
 663         setVectorizeEnable(true);
 664         break;
 665       case LoopHintAttr::Unroll:
 666         setUnrollState(LoopAttributes::Enable);
 667         break;
 668       case LoopHintAttr::UnrollAndJam:
 669         setUnrollAndJamState(LoopAttributes::Enable);
 670         break;
 671       case LoopHintAttr::VectorizePredicate:
 672         setVectorizePredicateState(LoopAttributes::Enable);
 673         break;
 674       case LoopHintAttr::Distribute:
 675         setDistributeState(true);
 676         break;
 677       case LoopHintAttr::UnrollCount:
 678       case LoopHintAttr::UnrollAndJamCount:
 679       case LoopHintAttr::VectorizeWidth:
 680       case LoopHintAttr::InterleaveCount:
 681       case LoopHintAttr::PipelineDisabled:
 682       case LoopHintAttr::PipelineInitiationInterval:
 683         llvm_unreachable("Options cannot enabled.");
 684         break;
 685       }
 686       break;
 687     case LoopHintAttr::AssumeSafety:
 688       switch (Option) {
 689       case LoopHintAttr::Vectorize:
 690       case LoopHintAttr::Interleave:
 691         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
 692         setParallel(true);
 693         setVectorizeEnable(true);
 694         break;
 695       case LoopHintAttr::Unroll:
 696       case LoopHintAttr::UnrollAndJam:
 697       case LoopHintAttr::VectorizePredicate:
 698       case LoopHintAttr::UnrollCount:
 699       case LoopHintAttr::UnrollAndJamCount:
 700       case LoopHintAttr::VectorizeWidth:
 701       case LoopHintAttr::InterleaveCount:
 702       case LoopHintAttr::Distribute:
 703       case LoopHintAttr::PipelineDisabled:
 704       case LoopHintAttr::PipelineInitiationInterval:
 705         llvm_unreachable("Options cannot be used to assume mem safety.");
 706         break;
 707       }
 708       break;
 709     case LoopHintAttr::Full:
 710       switch (Option) {
 711       case LoopHintAttr::Unroll:
 712         setUnrollState(LoopAttributes::Full);
 713         break;
 714       case LoopHintAttr::UnrollAndJam:
 715         setUnrollAndJamState(LoopAttributes::Full);
 716         break;
 717       case LoopHintAttr::Vectorize:
 718       case LoopHintAttr::Interleave:
 719       case LoopHintAttr::UnrollCount:
 720       case LoopHintAttr::UnrollAndJamCount:
 721       case LoopHintAttr::VectorizeWidth:
 722       case LoopHintAttr::InterleaveCount:
 723       case LoopHintAttr::Distribute:
 724       case LoopHintAttr::PipelineDisabled:
 725       case LoopHintAttr::PipelineInitiationInterval:
 726       case LoopHintAttr::VectorizePredicate:
 727         llvm_unreachable("Options cannot be used with 'full' hint.");
 728         break;
 729       }
 730       break;
 731     case LoopHintAttr::Numeric:
 732       switch (Option) {
 733       case LoopHintAttr::VectorizeWidth:
 734         setVectorizeWidth(ValueInt);
 735         break;
 736       case LoopHintAttr::InterleaveCount:
 737         setInterleaveCount(ValueInt);
 738         break;
 739       case LoopHintAttr::UnrollCount:
 740         setUnrollCount(ValueInt);
 741         break;
 742       case LoopHintAttr::UnrollAndJamCount:
 743         setUnrollAndJamCount(ValueInt);
 744         break;
 745       case LoopHintAttr::PipelineInitiationInterval:
 746         setPipelineInitiationInterval(ValueInt);
 747         break;
 748       case LoopHintAttr::Unroll:
 749       case LoopHintAttr::UnrollAndJam:
 750       case LoopHintAttr::VectorizePredicate:
 751       case LoopHintAttr::Vectorize:
 752       case LoopHintAttr::Interleave:
 753       case LoopHintAttr::Distribute:
 754       case LoopHintAttr::PipelineDisabled:
 755         llvm_unreachable("Options cannot be assigned a value.");
 756         break;
 757       }
 758       break;
 759     }
 760   }
 761
 762   /// Stage the attributes.
 763   push(Header, StartLoc, EndLoc);
 764 }
 765
 766 void LoopInfoStack::pop() {
 767   assert(!Active.empty() && "No active loops to pop");
 768   Active.back()->finish();
 769   Active.pop_back();
 770 }
 771
 772 void LoopInfoStack::InsertHelper(Instruction *I) const {
 773   if (I->mayReadOrWriteMemory()) {
 774     SmallVector<Metadata *, 4> AccessGroups;
 775     for (const auto &AL : Active) {
 776       // Here we assume that every loop that has an access group is parallel.
 777       if (MDNode *Group = AL->getAccessGroup())
 778         AccessGroups.push_back(Group);
 779     }
 780     MDNode *UnionMD = nullptr;
 781     if (AccessGroups.size() == 1)
 782       UnionMD = cast<MDNode>(AccessGroups[0]);
 783     else if (AccessGroups.size() >= 2)
 784       UnionMD = MDNode::get(I->getContext(), AccessGroups);
 785     I->setMetadata("llvm.access.group", UnionMD);
 786   }
 787
 788   if (!hasInfo())
 789     return;
 790
 791   const LoopInfo &L = getInfo();
 792   if (!L.getLoopID())
 793     return;
 794
 795   if (I->isTerminator()) {
 796     for (BasicBlock *Succ : successors(I))
 797       if (Succ == L.getHeader()) {
 798         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
 799         break;
 800       }
 801     return;
 802   }
 803 }