contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp

   1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "CGLoopInfo.h"
  10 #include "clang/AST/ASTContext.h"
  11 #include "clang/AST/Attr.h"
  12 #include "llvm/IR/BasicBlock.h"
  13 #include "llvm/IR/CFG.h"
  14 #include "llvm/IR/Constants.h"
  15 #include "llvm/IR/InstrTypes.h"
  16 #include "llvm/IR/Instructions.h"
  17 #include "llvm/IR/Metadata.h"
  18 using namespace clang::CodeGen;
  19 using namespace llvm;
  20
  21 MDNode *
  22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
  23   LLVMContext &Ctx = Header->getContext();
  24   SmallVector<Metadata *, 4> NewLoopProperties;
  25   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
  26   NewLoopProperties.push_back(TempNode.get());
  27   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
  28
  29   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
  30   LoopID->replaceOperandWith(0, LoopID);
  31   return LoopID;
  32 }
  33
  34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
  35                                            ArrayRef<Metadata *> LoopProperties,
  36                                            bool &HasUserTransforms) {
  37   LLVMContext &Ctx = Header->getContext();
  38
  39   Optional<bool> Enabled;
  40   if (Attrs.PipelineDisabled)
  41     Enabled = false;
  42   else if (Attrs.PipelineInitiationInterval != 0)
  43     Enabled = true;
  44
  45   if (Enabled != true) {
  46     SmallVector<Metadata *, 4> NewLoopProperties;
  47     if (Enabled == false) {
  48       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
  49       NewLoopProperties.push_back(
  50           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
  51                             ConstantAsMetadata::get(ConstantInt::get(
  52                                 llvm::Type::getInt1Ty(Ctx), 1))}));
  53       LoopProperties = NewLoopProperties;
  54     }
  55     return createLoopPropertiesMetadata(LoopProperties);
  56   }
  57
  58   SmallVector<Metadata *, 4> Args;
  59   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
  60   Args.push_back(TempNode.get());
  61   Args.append(LoopProperties.begin(), LoopProperties.end());
  62
  63   if (Attrs.PipelineInitiationInterval > 0) {
  64     Metadata *Vals[] = {
  65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
  66         ConstantAsMetadata::get(ConstantInt::get(
  67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
  68     Args.push_back(MDNode::get(Ctx, Vals));
  69   }
  70
  71   // No follow-up: This is the last transformation.
  72
  73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
  74   LoopID->replaceOperandWith(0, LoopID);
  75   HasUserTransforms = true;
  76   return LoopID;
  77 }
  78
  79 MDNode *
  80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
  81                                       ArrayRef<Metadata *> LoopProperties,
  82                                       bool &HasUserTransforms) {
  83   LLVMContext &Ctx = Header->getContext();
  84
  85   Optional<bool> Enabled;
  86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
  87     Enabled = false;
  88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
  89     Enabled = None;
  90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
  91            Attrs.UnrollCount != 0)
  92     Enabled = true;
  93
  94   if (Enabled != true) {
  95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
  96     // if unrolling is disabled.
  97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
  98   }
  99
 100   SmallVector<Metadata *, 4> FollowupLoopProperties;
 101
 102   // Apply all loop properties to the unrolled loop.
 103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 104
 105   // Don't unroll an already unrolled loop.
 106   FollowupLoopProperties.push_back(
 107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
 108
 109   bool FollowupHasTransforms = false;
 110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
 111                                               FollowupHasTransforms);
 112
 113   SmallVector<Metadata *, 4> Args;
 114   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 115   Args.push_back(TempNode.get());
 116   Args.append(LoopProperties.begin(), LoopProperties.end());
 117
 118   // Setting unroll.count
 119   if (Attrs.UnrollCount > 0) {
 120     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
 121                         ConstantAsMetadata::get(ConstantInt::get(
 122                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
 123     Args.push_back(MDNode::get(Ctx, Vals));
 124   }
 125
 126   // Setting unroll.full or unroll.disable
 127   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
 128     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
 129     Args.push_back(MDNode::get(Ctx, Vals));
 130   }
 131
 132   if (FollowupHasTransforms)
 133     Args.push_back(MDNode::get(
 134         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
 135
 136   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
 137   LoopID->replaceOperandWith(0, LoopID);
 138   HasUserTransforms = true;
 139   return LoopID;
 140 }
 141
 142 MDNode *
 143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
 144                                      ArrayRef<Metadata *> LoopProperties,
 145                                      bool &HasUserTransforms) {
 146   LLVMContext &Ctx = Header->getContext();
 147
 148   Optional<bool> Enabled;
 149   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
 150     Enabled = false;
 151   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
 152            Attrs.UnrollAndJamCount != 0)
 153     Enabled = true;
 154
 155   if (Enabled != true) {
 156     SmallVector<Metadata *, 4> NewLoopProperties;
 157     if (Enabled == false) {
 158       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 159       NewLoopProperties.push_back(MDNode::get(
 160           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
 161       LoopProperties = NewLoopProperties;
 162     }
 163     return createPartialUnrollMetadata(Attrs, LoopProperties,
 164                                        HasUserTransforms);
 165   }
 166
 167   SmallVector<Metadata *, 4> FollowupLoopProperties;
 168   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 169   FollowupLoopProperties.push_back(
 170       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
 171
 172   bool FollowupHasTransforms = false;
 173   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
 174                                                  FollowupHasTransforms);
 175
 176   SmallVector<Metadata *, 4> Args;
 177   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 178   Args.push_back(TempNode.get());
 179   Args.append(LoopProperties.begin(), LoopProperties.end());
 180
 181   // Setting unroll_and_jam.count
 182   if (Attrs.UnrollAndJamCount > 0) {
 183     Metadata *Vals[] = {
 184         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
 185         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
 186                                                  Attrs.UnrollAndJamCount))};
 187     Args.push_back(MDNode::get(Ctx, Vals));
 188   }
 189
 190   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
 191     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
 192     Args.push_back(MDNode::get(Ctx, Vals));
 193   }
 194
 195   if (FollowupHasTransforms)
 196     Args.push_back(MDNode::get(
 197         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
 198               Followup}));
 199
 200   if (UnrollAndJamInnerFollowup)
 201     Args.push_back(MDNode::get(
 202         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
 203               UnrollAndJamInnerFollowup}));
 204
 205   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
 206   LoopID->replaceOperandWith(0, LoopID);
 207   HasUserTransforms = true;
 208   return LoopID;
 209 }
 210
 211 MDNode *
 212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
 213                                       ArrayRef<Metadata *> LoopProperties,
 214                                       bool &HasUserTransforms) {
 215   LLVMContext &Ctx = Header->getContext();
 216
 217   Optional<bool> Enabled;
 218   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
 219     Enabled = false;
 220   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
 221            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
 222            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
 223     Enabled = true;
 224
 225   if (Enabled != true) {
 226     SmallVector<Metadata *, 4> NewLoopProperties;
 227     if (Enabled == false) {
 228       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 229       NewLoopProperties.push_back(
 230           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
 231                             ConstantAsMetadata::get(ConstantInt::get(
 232                                 llvm::Type::getInt1Ty(Ctx), 0))}));
 233       LoopProperties = NewLoopProperties;
 234     }
 235     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
 236   }
 237
 238   // Apply all loop properties to the vectorized loop.
 239   SmallVector<Metadata *, 4> FollowupLoopProperties;
 240   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 241
 242   // Don't vectorize an already vectorized loop.
 243   FollowupLoopProperties.push_back(
 244       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
 245
 246   bool FollowupHasTransforms = false;
 247   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
 248                                                 FollowupHasTransforms);
 249
 250   SmallVector<Metadata *, 4> Args;
 251   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 252   Args.push_back(TempNode.get());
 253   Args.append(LoopProperties.begin(), LoopProperties.end());
 254
 255   // Setting vectorize.predicate
 256   bool IsVectorPredicateEnabled = false;
 257   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
 258       Attrs.VectorizeEnable != LoopAttributes::Disable &&
 259       Attrs.VectorizeWidth < 1) {
 260
 261     IsVectorPredicateEnabled =
 262         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
 263
 264     Metadata *Vals[] = {
 265         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
 266         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
 267                                                  IsVectorPredicateEnabled))};
 268     Args.push_back(MDNode::get(Ctx, Vals));
 269   }
 270
 271   // Setting vectorize.width
 272   if (Attrs.VectorizeWidth > 0) {
 273     Metadata *Vals[] = {
 274         MDString::get(Ctx, "llvm.loop.vectorize.width"),
 275         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
 276                                                  Attrs.VectorizeWidth))};
 277     Args.push_back(MDNode::get(Ctx, Vals));
 278   }
 279
 280   // Setting interleave.count
 281   if (Attrs.InterleaveCount > 0) {
 282     Metadata *Vals[] = {
 283         MDString::get(Ctx, "llvm.loop.interleave.count"),
 284         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
 285                                                  Attrs.InterleaveCount))};
 286     Args.push_back(MDNode::get(Ctx, Vals));
 287   }
 288
 289   // vectorize.enable is set if:
 290   // 1) loop hint vectorize.enable is set, or
 291   // 2) it is implied when vectorize.predicate is set, or
 292   // 3) it is implied when vectorize.width is set.
 293   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
 294       IsVectorPredicateEnabled ||
 295       Attrs.VectorizeWidth > 1 ) {
 296     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
 297     Args.push_back(
 298         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
 299                           ConstantAsMetadata::get(ConstantInt::get(
 300                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
 301   }
 302
 303   if (FollowupHasTransforms)
 304     Args.push_back(MDNode::get(
 305         Ctx,
 306         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
 307
 308   MDNode *LoopID = MDNode::get(Ctx, Args);
 309   LoopID->replaceOperandWith(0, LoopID);
 310   HasUserTransforms = true;
 311   return LoopID;
 312 }
 313
 314 MDNode *
 315 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
 316                                        ArrayRef<Metadata *> LoopProperties,
 317                                        bool &HasUserTransforms) {
 318   LLVMContext &Ctx = Header->getContext();
 319
 320   Optional<bool> Enabled;
 321   if (Attrs.DistributeEnable == LoopAttributes::Disable)
 322     Enabled = false;
 323   if (Attrs.DistributeEnable == LoopAttributes::Enable)
 324     Enabled = true;
 325
 326   if (Enabled != true) {
 327     SmallVector<Metadata *, 4> NewLoopProperties;
 328     if (Enabled == false) {
 329       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 330       NewLoopProperties.push_back(
 331           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
 332                             ConstantAsMetadata::get(ConstantInt::get(
 333                                 llvm::Type::getInt1Ty(Ctx), 0))}));
 334       LoopProperties = NewLoopProperties;
 335     }
 336     return createLoopVectorizeMetadata(Attrs, LoopProperties,
 337                                        HasUserTransforms);
 338   }
 339
 340   bool FollowupHasTransforms = false;
 341   MDNode *Followup =
 342       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
 343
 344   SmallVector<Metadata *, 4> Args;
 345   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 346   Args.push_back(TempNode.get());
 347   Args.append(LoopProperties.begin(), LoopProperties.end());
 348
 349   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
 350                       ConstantAsMetadata::get(ConstantInt::get(
 351                           llvm::Type::getInt1Ty(Ctx),
 352                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
 353   Args.push_back(MDNode::get(Ctx, Vals));
 354
 355   if (FollowupHasTransforms)
 356     Args.push_back(MDNode::get(
 357         Ctx,
 358         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
 359
 360   MDNode *LoopID = MDNode::get(Ctx, Args);
 361   LoopID->replaceOperandWith(0, LoopID);
 362   HasUserTransforms = true;
 363   return LoopID;
 364 }
 365
 366 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
 367                                            ArrayRef<Metadata *> LoopProperties,
 368                                            bool &HasUserTransforms) {
 369   LLVMContext &Ctx = Header->getContext();
 370
 371   Optional<bool> Enabled;
 372   if (Attrs.UnrollEnable == LoopAttributes::Disable)
 373     Enabled = false;
 374   else if (Attrs.UnrollEnable == LoopAttributes::Full)
 375     Enabled = true;
 376
 377   if (Enabled != true) {
 378     SmallVector<Metadata *, 4> NewLoopProperties;
 379     if (Enabled == false) {
 380       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
 381       NewLoopProperties.push_back(
 382           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
 383       LoopProperties = NewLoopProperties;
 384     }
 385     return createLoopDistributeMetadata(Attrs, LoopProperties,
 386                                         HasUserTransforms);
 387   }
 388
 389   SmallVector<Metadata *, 4> Args;
 390   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
 391   Args.push_back(TempNode.get());
 392   Args.append(LoopProperties.begin(), LoopProperties.end());
 393   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
 394
 395   // No follow-up: there is no loop after full unrolling.
 396   // TODO: Warn if there are transformations after full unrolling.
 397
 398   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
 399   LoopID->replaceOperandWith(0, LoopID);
 400   HasUserTransforms = true;
 401   return LoopID;
 402 }
 403
 404 MDNode *LoopInfo::createMetadata(
 405     const LoopAttributes &Attrs,
 406     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
 407     bool &HasUserTransforms) {
 408   SmallVector<Metadata *, 3> LoopProperties;
 409
 410   // If we have a valid start debug location for the loop, add it.
 411   if (StartLoc) {
 412     LoopProperties.push_back(StartLoc.getAsMDNode());
 413
 414     // If we also have a valid end debug location for the loop, add it.
 415     if (EndLoc)
 416       LoopProperties.push_back(EndLoc.getAsMDNode());
 417   }
 418
 419   assert(!!AccGroup == Attrs.IsParallel &&
 420          "There must be an access group iff the loop is parallel");
 421   if (Attrs.IsParallel) {
 422     LLVMContext &Ctx = Header->getContext();
 423     LoopProperties.push_back(MDNode::get(
 424         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
 425   }
 426
 427   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
 428                         AdditionalLoopProperties.end());
 429   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
 430 }
 431
 432 LoopAttributes::LoopAttributes(bool IsParallel)
 433     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
 434       UnrollEnable(LoopAttributes::Unspecified),
 435       UnrollAndJamEnable(LoopAttributes::Unspecified),
 436       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
 437       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
 438       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
 439       PipelineInitiationInterval(0) {}
 440
 441 void LoopAttributes::clear() {
 442   IsParallel = false;
 443   VectorizeWidth = 0;
 444   InterleaveCount = 0;
 445   UnrollCount = 0;
 446   UnrollAndJamCount = 0;
 447   VectorizeEnable = LoopAttributes::Unspecified;
 448   UnrollEnable = LoopAttributes::Unspecified;
 449   UnrollAndJamEnable = LoopAttributes::Unspecified;
 450   VectorizePredicateEnable = LoopAttributes::Unspecified;
 451   DistributeEnable = LoopAttributes::Unspecified;
 452   PipelineDisabled = false;
 453   PipelineInitiationInterval = 0;
 454 }
 455
 456 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
 457                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
 458                    LoopInfo *Parent)
 459     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
 460       Parent(Parent) {
 461
 462   if (Attrs.IsParallel) {
 463     // Create an access group for this loop.
 464     LLVMContext &Ctx = Header->getContext();
 465     AccGroup = MDNode::getDistinct(Ctx, {});
 466   }
 467
 468   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
 469       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
 470       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
 471       Attrs.PipelineInitiationInterval == 0 &&
 472       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
 473       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
 474       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
 475       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
 476       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
 477       !EndLoc)
 478     return;
 479
 480   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
 481 }
 482
 483 void LoopInfo::finish() {
 484   // We did not annotate the loop body instructions because there are no
 485   // attributes for this loop.
 486   if (!TempLoopID)
 487     return;
 488
 489   MDNode *LoopID;
 490   LoopAttributes CurLoopAttr = Attrs;
 491   LLVMContext &Ctx = Header->getContext();
 492
 493   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
 494                  Parent->Attrs.UnrollAndJamCount != 0)) {
 495     // Parent unroll-and-jams this loop.
 496     // Split the transformations in those that happens before the unroll-and-jam
 497     // and those after.
 498
 499     LoopAttributes BeforeJam, AfterJam;
 500
 501     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
 502
 503     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
 504     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
 505     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
 506     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
 507     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
 508
 509     switch (Attrs.UnrollEnable) {
 510     case LoopAttributes::Unspecified:
 511     case LoopAttributes::Disable:
 512       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
 513       AfterJam.UnrollEnable = Attrs.UnrollEnable;
 514       break;
 515     case LoopAttributes::Full:
 516       BeforeJam.UnrollEnable = LoopAttributes::Full;
 517       break;
 518     case LoopAttributes::Enable:
 519       AfterJam.UnrollEnable = LoopAttributes::Enable;
 520       break;
 521     }
 522
 523     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
 524     AfterJam.UnrollCount = Attrs.UnrollCount;
 525     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
 526     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
 527
 528     // If this loop is subject of an unroll-and-jam by the parent loop, and has
 529     // an unroll-and-jam annotation itself, we have to decide whether to first
 530     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
 531     // UnrollAndJam pass processes loops from inner to outer, so we apply the
 532     // inner first.
 533     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
 534     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
 535
 536     // Set the inner followup metadata to process by the outer loop. Only
 537     // consider the first inner loop.
 538     if (!Parent->UnrollAndJamInnerFollowup) {
 539       // Splitting the attributes into a BeforeJam and an AfterJam part will
 540       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
 541       // to be forwarded to the AfterJam part. We detect the situation here and
 542       // add it manually.
 543       SmallVector<Metadata *, 1> BeforeLoopProperties;
 544       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
 545           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
 546           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
 547         BeforeLoopProperties.push_back(
 548             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
 549
 550       bool InnerFollowupHasTransform = false;
 551       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
 552                                              InnerFollowupHasTransform);
 553       if (InnerFollowupHasTransform)
 554         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
 555     }
 556
 557     CurLoopAttr = BeforeJam;
 558   }
 559
 560   bool HasUserTransforms = false;
 561   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
 562   TempLoopID->replaceAllUsesWith(LoopID);
 563 }
 564
 565 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
 566                          const llvm::DebugLoc &EndLoc) {
 567   Active.emplace_back(
 568       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
 569                    Active.empty() ? nullptr : Active.back().get()));
 570   // Clear the attributes so nested loops do not inherit them.
 571   StagedAttrs.clear();
 572 }
 573
 574 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
 575                          ArrayRef<const clang::Attr *> Attrs,
 576                          const llvm::DebugLoc &StartLoc,
 577                          const llvm::DebugLoc &EndLoc) {
 578
 579   // Identify loop hint attributes from Attrs.
 580   for (const auto *Attr : Attrs) {
 581     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
 582     const OpenCLUnrollHintAttr *OpenCLHint =
 583         dyn_cast<OpenCLUnrollHintAttr>(Attr);
 584
 585     // Skip non loop hint attributes
 586     if (!LH && !OpenCLHint) {
 587       continue;
 588     }
 589
 590     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
 591     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
 592     unsigned ValueInt = 1;
 593     // Translate opencl_unroll_hint attribute argument to
 594     // equivalent LoopHintAttr enums.
 595     // OpenCL v2.0 s6.11.5:
 596     // 0 - enable unroll (no argument).
 597     // 1 - disable unroll.
 598     // other positive integer n - unroll by n.
 599     if (OpenCLHint) {
 600       ValueInt = OpenCLHint->getUnrollHint();
 601       if (ValueInt == 0) {
 602         State = LoopHintAttr::Enable;
 603       } else if (ValueInt != 1) {
 604         Option = LoopHintAttr::UnrollCount;
 605         State = LoopHintAttr::Numeric;
 606       }
 607     } else if (LH) {
 608       auto *ValueExpr = LH->getValue();
 609       if (ValueExpr) {
 610         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
 611         ValueInt = ValueAPS.getSExtValue();
 612       }
 613
 614       Option = LH->getOption();
 615       State = LH->getState();
 616     }
 617     switch (State) {
 618     case LoopHintAttr::Disable:
 619       switch (Option) {
 620       case LoopHintAttr::Vectorize:
 621         // Disable vectorization by specifying a width of 1.
 622         setVectorizeWidth(1);
 623         break;
 624       case LoopHintAttr::Interleave:
 625         // Disable interleaving by speciyfing a count of 1.
 626         setInterleaveCount(1);
 627         break;
 628       case LoopHintAttr::Unroll:
 629         setUnrollState(LoopAttributes::Disable);
 630         break;
 631       case LoopHintAttr::UnrollAndJam:
 632         setUnrollAndJamState(LoopAttributes::Disable);
 633         break;
 634       case LoopHintAttr::VectorizePredicate:
 635         setVectorizePredicateState(LoopAttributes::Disable);
 636         break;
 637       case LoopHintAttr::Distribute:
 638         setDistributeState(false);
 639         break;
 640       case LoopHintAttr::PipelineDisabled:
 641         setPipelineDisabled(true);
 642         break;
 643       case LoopHintAttr::UnrollCount:
 644       case LoopHintAttr::UnrollAndJamCount:
 645       case LoopHintAttr::VectorizeWidth:
 646       case LoopHintAttr::InterleaveCount:
 647       case LoopHintAttr::PipelineInitiationInterval:
 648         llvm_unreachable("Options cannot be disabled.");
 649         break;
 650       }
 651       break;
 652     case LoopHintAttr::Enable:
 653       switch (Option) {
 654       case LoopHintAttr::Vectorize:
 655       case LoopHintAttr::Interleave:
 656         setVectorizeEnable(true);
 657         break;
 658       case LoopHintAttr::Unroll:
 659         setUnrollState(LoopAttributes::Enable);
 660         break;
 661       case LoopHintAttr::UnrollAndJam:
 662         setUnrollAndJamState(LoopAttributes::Enable);
 663         break;
 664       case LoopHintAttr::VectorizePredicate:
 665         setVectorizePredicateState(LoopAttributes::Enable);
 666         break;
 667       case LoopHintAttr::Distribute:
 668         setDistributeState(true);
 669         break;
 670       case LoopHintAttr::UnrollCount:
 671       case LoopHintAttr::UnrollAndJamCount:
 672       case LoopHintAttr::VectorizeWidth:
 673       case LoopHintAttr::InterleaveCount:
 674       case LoopHintAttr::PipelineDisabled:
 675       case LoopHintAttr::PipelineInitiationInterval:
 676         llvm_unreachable("Options cannot enabled.");
 677         break;
 678       }
 679       break;
 680     case LoopHintAttr::AssumeSafety:
 681       switch (Option) {
 682       case LoopHintAttr::Vectorize:
 683       case LoopHintAttr::Interleave:
 684         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
 685         setParallel(true);
 686         setVectorizeEnable(true);
 687         break;
 688       case LoopHintAttr::Unroll:
 689       case LoopHintAttr::UnrollAndJam:
 690       case LoopHintAttr::VectorizePredicate:
 691       case LoopHintAttr::UnrollCount:
 692       case LoopHintAttr::UnrollAndJamCount:
 693       case LoopHintAttr::VectorizeWidth:
 694       case LoopHintAttr::InterleaveCount:
 695       case LoopHintAttr::Distribute:
 696       case LoopHintAttr::PipelineDisabled:
 697       case LoopHintAttr::PipelineInitiationInterval:
 698         llvm_unreachable("Options cannot be used to assume mem safety.");
 699         break;
 700       }
 701       break;
 702     case LoopHintAttr::Full:
 703       switch (Option) {
 704       case LoopHintAttr::Unroll:
 705         setUnrollState(LoopAttributes::Full);
 706         break;
 707       case LoopHintAttr::UnrollAndJam:
 708         setUnrollAndJamState(LoopAttributes::Full);
 709         break;
 710       case LoopHintAttr::Vectorize:
 711       case LoopHintAttr::Interleave:
 712       case LoopHintAttr::UnrollCount:
 713       case LoopHintAttr::UnrollAndJamCount:
 714       case LoopHintAttr::VectorizeWidth:
 715       case LoopHintAttr::InterleaveCount:
 716       case LoopHintAttr::Distribute:
 717       case LoopHintAttr::PipelineDisabled:
 718       case LoopHintAttr::PipelineInitiationInterval:
 719       case LoopHintAttr::VectorizePredicate:
 720         llvm_unreachable("Options cannot be used with 'full' hint.");
 721         break;
 722       }
 723       break;
 724     case LoopHintAttr::Numeric:
 725       switch (Option) {
 726       case LoopHintAttr::VectorizeWidth:
 727         setVectorizeWidth(ValueInt);
 728         break;
 729       case LoopHintAttr::InterleaveCount:
 730         setInterleaveCount(ValueInt);
 731         break;
 732       case LoopHintAttr::UnrollCount:
 733         setUnrollCount(ValueInt);
 734         break;
 735       case LoopHintAttr::UnrollAndJamCount:
 736         setUnrollAndJamCount(ValueInt);
 737         break;
 738       case LoopHintAttr::PipelineInitiationInterval:
 739         setPipelineInitiationInterval(ValueInt);
 740         break;
 741       case LoopHintAttr::Unroll:
 742       case LoopHintAttr::UnrollAndJam:
 743       case LoopHintAttr::VectorizePredicate:
 744       case LoopHintAttr::Vectorize:
 745       case LoopHintAttr::Interleave:
 746       case LoopHintAttr::Distribute:
 747       case LoopHintAttr::PipelineDisabled:
 748         llvm_unreachable("Options cannot be assigned a value.");
 749         break;
 750       }
 751       break;
 752     }
 753   }
 754
 755   /// Stage the attributes.
 756   push(Header, StartLoc, EndLoc);
 757 }
 758
 759 void LoopInfoStack::pop() {
 760   assert(!Active.empty() && "No active loops to pop");
 761   Active.back()->finish();
 762   Active.pop_back();
 763 }
 764
 765 void LoopInfoStack::InsertHelper(Instruction *I) const {
 766   if (I->mayReadOrWriteMemory()) {
 767     SmallVector<Metadata *, 4> AccessGroups;
 768     for (const auto &AL : Active) {
 769       // Here we assume that every loop that has an access group is parallel.
 770       if (MDNode *Group = AL->getAccessGroup())
 771         AccessGroups.push_back(Group);
 772     }
 773     MDNode *UnionMD = nullptr;
 774     if (AccessGroups.size() == 1)
 775       UnionMD = cast<MDNode>(AccessGroups[0]);
 776     else if (AccessGroups.size() >= 2)
 777       UnionMD = MDNode::get(I->getContext(), AccessGroups);
 778     I->setMetadata("llvm.access.group", UnionMD);
 779   }
 780
 781   if (!hasInfo())
 782     return;
 783
 784   const LoopInfo &L = getInfo();
 785   if (!L.getLoopID())
 786     return;
 787
 788   if (I->isTerminator()) {
 789     for (BasicBlock *Succ : successors(I))
 790       if (Succ == L.getHeader()) {
 791         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
 792         break;
 793       }
 794     return;
 795   }
 796 }