lib/CodeGen/CGOpenMPRuntime.cpp

   1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This provides a class for OpenMP runtime code generation.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "CGCXXABI.h"
  15 #include "CGCleanup.h"
  16 #include "CGOpenMPRuntime.h"
  17 #include "CGRecordLayout.h"
  18 #include "CodeGenFunction.h"
  19 #include "clang/CodeGen/ConstantInitBuilder.h"
  20 #include "clang/AST/Decl.h"
  21 #include "clang/AST/StmtOpenMP.h"
  22 #include "clang/Basic/BitmaskEnum.h"
  23 #include "llvm/ADT/ArrayRef.h"
  24 #include "llvm/Bitcode/BitcodeReader.h"
  25 #include "llvm/IR/CallSite.h"
  26 #include "llvm/IR/DerivedTypes.h"
  27 #include "llvm/IR/GlobalValue.h"
  28 #include "llvm/IR/Value.h"
  29 #include "llvm/Support/Format.h"
  30 #include "llvm/Support/raw_ostream.h"
  31 #include <cassert>
  32
  33 using namespace clang;
  34 using namespace CodeGen;
  35
  36 namespace {
  37 /// Base class for handling code generation inside OpenMP regions.
  38 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
  39 public:
  40   /// Kinds of OpenMP regions used in codegen.
  41   enum CGOpenMPRegionKind {
  42     /// Region with outlined function for standalone 'parallel'
  43     /// directive.
  44     ParallelOutlinedRegion,
  45     /// Region with outlined function for standalone 'task' directive.
  46     TaskOutlinedRegion,
  47     /// Region for constructs that do not require function outlining,
  48     /// like 'for', 'sections', 'atomic' etc. directives.
  49     InlinedRegion,
  50     /// Region with outlined function for standalone 'target' directive.
  51     TargetRegion,
  52   };
  53
  54   CGOpenMPRegionInfo(const CapturedStmt &CS,
  55                      const CGOpenMPRegionKind RegionKind,
  56                      const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
  57                      bool HasCancel)
  58       : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
  59         CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
  60
  61   CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
  62                      const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
  63                      bool HasCancel)
  64       : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
  65         Kind(Kind), HasCancel(HasCancel) {}
  66
  67   /// Get a variable or parameter for storing global thread id
  68   /// inside OpenMP construct.
  69   virtual const VarDecl *getThreadIDVariable() const = 0;
  70
  71   /// Emit the captured statement body.
  72   void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
  73
  74   /// Get an LValue for the current ThreadID variable.
  75   /// \return LValue for thread id variable. This LValue always has type int32*.
  76   virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
  77
  78   virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
  79
  80   CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
  81
  82   OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
  83
  84   bool hasCancel() const { return HasCancel; }
  85
  86   static bool classof(const CGCapturedStmtInfo *Info) {
  87     return Info->getKind() == CR_OpenMP;
  88   }
  89
  90   ~CGOpenMPRegionInfo() override = default;
  91
  92 protected:
  93   CGOpenMPRegionKind RegionKind;
  94   RegionCodeGenTy CodeGen;
  95   OpenMPDirectiveKind Kind;
  96   bool HasCancel;
  97 };
  98
  99 /// API for captured statement code generation in OpenMP constructs.
 100 class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
 101 public:
 102   CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
 103                              const RegionCodeGenTy &CodeGen,
 104                              OpenMPDirectiveKind Kind, bool HasCancel,
 105                              StringRef HelperName)
 106       : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
 107                            HasCancel),
 108         ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
 109     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
 110   }
 111
 112   /// Get a variable or parameter for storing global thread id
 113   /// inside OpenMP construct.
 114   const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
 115
 116   /// Get the name of the capture helper.
 117   StringRef getHelperName() const override { return HelperName; }
 118
 119   static bool classof(const CGCapturedStmtInfo *Info) {
 120     return CGOpenMPRegionInfo::classof(Info) &&
 121            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
 122                ParallelOutlinedRegion;
 123   }
 124
 125 private:
 126   /// A variable or parameter storing global thread id for OpenMP
 127   /// constructs.
 128   const VarDecl *ThreadIDVar;
 129   StringRef HelperName;
 130 };
 131
 132 /// API for captured statement code generation in OpenMP constructs.
 133 class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
 134 public:
 135   class UntiedTaskActionTy final : public PrePostActionTy {
 136     bool Untied;
 137     const VarDecl *PartIDVar;
 138     const RegionCodeGenTy UntiedCodeGen;
 139     llvm::SwitchInst *UntiedSwitch = nullptr;
 140
 141   public:
 142     UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
 143                        const RegionCodeGenTy &UntiedCodeGen)
 144         : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
 145     void Enter(CodeGenFunction &CGF) override {
 146       if (Untied) {
 147         // Emit task switching point.
 148         LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
 149             CGF.GetAddrOfLocalVar(PartIDVar),
 150             PartIDVar->getType()->castAs<PointerType>());
 151         llvm::Value *Res =
 152             CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation());
 153         llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done.");
 154         UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB);
 155         CGF.EmitBlock(DoneBB);
 156         CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
 157         CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
 158         UntiedSwitch->addCase(CGF.Builder.getInt32(0),
 159                               CGF.Builder.GetInsertBlock());
 160         emitUntiedSwitch(CGF);
 161       }
 162     }
 163     void emitUntiedSwitch(CodeGenFunction &CGF) const {
 164       if (Untied) {
 165         LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
 166             CGF.GetAddrOfLocalVar(PartIDVar),
 167             PartIDVar->getType()->castAs<PointerType>());
 168         CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
 169                               PartIdLVal);
 170         UntiedCodeGen(CGF);
 171         CodeGenFunction::JumpDest CurPoint =
 172             CGF.getJumpDestInCurrentScope(".untied.next.");
 173         CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
 174         CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp."));
 175         UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()),
 176                               CGF.Builder.GetInsertBlock());
 177         CGF.EmitBranchThroughCleanup(CurPoint);
 178         CGF.EmitBlock(CurPoint.getBlock());
 179       }
 180     }
 181     unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
 182   };
 183   CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
 184                                  const VarDecl *ThreadIDVar,
 185                                  const RegionCodeGenTy &CodeGen,
 186                                  OpenMPDirectiveKind Kind, bool HasCancel,
 187                                  const UntiedTaskActionTy &Action)
 188       : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
 189         ThreadIDVar(ThreadIDVar), Action(Action) {
 190     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
 191   }
 192
 193   /// Get a variable or parameter for storing global thread id
 194   /// inside OpenMP construct.
 195   const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
 196
 197   /// Get an LValue for the current ThreadID variable.
 198   LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
 199
 200   /// Get the name of the capture helper.
 201   StringRef getHelperName() const override { return ".omp_outlined."; }
 202
 203   void emitUntiedSwitch(CodeGenFunction &CGF) override {
 204     Action.emitUntiedSwitch(CGF);
 205   }
 206
 207   static bool classof(const CGCapturedStmtInfo *Info) {
 208     return CGOpenMPRegionInfo::classof(Info) &&
 209            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
 210                TaskOutlinedRegion;
 211   }
 212
 213 private:
 214   /// A variable or parameter storing global thread id for OpenMP
 215   /// constructs.
 216   const VarDecl *ThreadIDVar;
 217   /// Action for emitting code for untied tasks.
 218   const UntiedTaskActionTy &Action;
 219 };
 220
 221 /// API for inlined captured statement code generation in OpenMP
 222 /// constructs.
 223 class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
 224 public:
 225   CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
 226                             const RegionCodeGenTy &CodeGen,
 227                             OpenMPDirectiveKind Kind, bool HasCancel)
 228       : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
 229         OldCSI(OldCSI),
 230         OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
 231
 232   // Retrieve the value of the context parameter.
 233   llvm::Value *getContextValue() const override {
 234     if (OuterRegionInfo)
 235       return OuterRegionInfo->getContextValue();
 236     llvm_unreachable("No context value for inlined OpenMP region");
 237   }
 238
 239   void setContextValue(llvm::Value *V) override {
 240     if (OuterRegionInfo) {
 241       OuterRegionInfo->setContextValue(V);
 242       return;
 243     }
 244     llvm_unreachable("No context value for inlined OpenMP region");
 245   }
 246
 247   /// Lookup the captured field decl for a variable.
 248   const FieldDecl *lookup(const VarDecl *VD) const override {
 249     if (OuterRegionInfo)
 250       return OuterRegionInfo->lookup(VD);
 251     // If there is no outer outlined region,no need to lookup in a list of
 252     // captured variables, we can use the original one.
 253     return nullptr;
 254   }
 255
 256   FieldDecl *getThisFieldDecl() const override {
 257     if (OuterRegionInfo)
 258       return OuterRegionInfo->getThisFieldDecl();
 259     return nullptr;
 260   }
 261
 262   /// Get a variable or parameter for storing global thread id
 263   /// inside OpenMP construct.
 264   const VarDecl *getThreadIDVariable() const override {
 265     if (OuterRegionInfo)
 266       return OuterRegionInfo->getThreadIDVariable();
 267     return nullptr;
 268   }
 269
 270   /// Get an LValue for the current ThreadID variable.
 271   LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
 272     if (OuterRegionInfo)
 273       return OuterRegionInfo->getThreadIDVariableLValue(CGF);
 274     llvm_unreachable("No LValue for inlined OpenMP construct");
 275   }
 276
 277   /// Get the name of the capture helper.
 278   StringRef getHelperName() const override {
 279     if (auto *OuterRegionInfo = getOldCSI())
 280       return OuterRegionInfo->getHelperName();
 281     llvm_unreachable("No helper name for inlined OpenMP construct");
 282   }
 283
 284   void emitUntiedSwitch(CodeGenFunction &CGF) override {
 285     if (OuterRegionInfo)
 286       OuterRegionInfo->emitUntiedSwitch(CGF);
 287   }
 288
 289   CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
 290
 291   static bool classof(const CGCapturedStmtInfo *Info) {
 292     return CGOpenMPRegionInfo::classof(Info) &&
 293            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
 294   }
 295
 296   ~CGOpenMPInlinedRegionInfo() override = default;
 297
 298 private:
 299   /// CodeGen info about outer OpenMP region.
 300   CodeGenFunction::CGCapturedStmtInfo *OldCSI;
 301   CGOpenMPRegionInfo *OuterRegionInfo;
 302 };
 303
 304 /// API for captured statement code generation in OpenMP target
 305 /// constructs. For this captures, implicit parameters are used instead of the
 306 /// captured fields. The name of the target region has to be unique in a given
 307 /// application so it is provided by the client, because only the client has
 308 /// the information to generate that.
 309 class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
 310 public:
 311   CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
 312                            const RegionCodeGenTy &CodeGen, StringRef HelperName)
 313       : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
 314                            /*HasCancel=*/false),
 315         HelperName(HelperName) {}
 316
 317   /// This is unused for target regions because each starts executing
 318   /// with a single thread.
 319   const VarDecl *getThreadIDVariable() const override { return nullptr; }
 320
 321   /// Get the name of the capture helper.
 322   StringRef getHelperName() const override { return HelperName; }
 323
 324   static bool classof(const CGCapturedStmtInfo *Info) {
 325     return CGOpenMPRegionInfo::classof(Info) &&
 326            cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
 327   }
 328
 329 private:
 330   StringRef HelperName;
 331 };
 332
 333 static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
 334   llvm_unreachable("No codegen for expressions");
 335 }
 336 /// API for generation of expressions captured in a innermost OpenMP
 337 /// region.
 338 class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
 339 public:
 340   CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
 341       : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
 342                                   OMPD_unknown,
 343                                   /*HasCancel=*/false),
 344         PrivScope(CGF) {
 345     // Make sure the globals captured in the provided statement are local by
 346     // using the privatization logic. We assume the same variable is not
 347     // captured more than once.
 348     for (const auto &C : CS.captures()) {
 349       if (!C.capturesVariable() && !C.capturesVariableByCopy())
 350         continue;
 351
 352       const VarDecl *VD = C.getCapturedVar();
 353       if (VD->isLocalVarDeclOrParm())
 354         continue;
 355
 356       DeclRefExpr DRE(const_cast<VarDecl *>(VD),
 357                       /*RefersToEnclosingVariableOrCapture=*/false,
 358                       VD->getType().getNonReferenceType(), VK_LValue,
 359                       C.getLocation());
 360       PrivScope.addPrivate(
 361           VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(); });
 362     }
 363     (void)PrivScope.Privatize();
 364   }
 365
 366   /// Lookup the captured field decl for a variable.
 367   const FieldDecl *lookup(const VarDecl *VD) const override {
 368     if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
 369       return FD;
 370     return nullptr;
 371   }
 372
 373   /// Emit the captured statement body.
 374   void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
 375     llvm_unreachable("No body for expressions");
 376   }
 377
 378   /// Get a variable or parameter for storing global thread id
 379   /// inside OpenMP construct.
 380   const VarDecl *getThreadIDVariable() const override {
 381     llvm_unreachable("No thread id for expressions");
 382   }
 383
 384   /// Get the name of the capture helper.
 385   StringRef getHelperName() const override {
 386     llvm_unreachable("No helper name for expressions");
 387   }
 388
 389   static bool classof(const CGCapturedStmtInfo *Info) { return false; }
 390
 391 private:
 392   /// Private scope to capture global variables.
 393   CodeGenFunction::OMPPrivateScope PrivScope;
 394 };
 395
 396 /// RAII for emitting code of OpenMP constructs.
 397 class InlinedOpenMPRegionRAII {
 398   CodeGenFunction &CGF;
 399   llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
 400   FieldDecl *LambdaThisCaptureField = nullptr;
 401   const CodeGen::CGBlockInfo *BlockInfo = nullptr;
 402
 403 public:
 404   /// Constructs region for combined constructs.
 405   /// \param CodeGen Code generation sequence for combined directives. Includes
 406   /// a list of functions used for code generation of implicitly inlined
 407   /// regions.
 408   InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
 409                           OpenMPDirectiveKind Kind, bool HasCancel)
 410       : CGF(CGF) {
 411     // Start emission for the construct.
 412     CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
 413         CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
 414     std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
 415     LambdaThisCaptureField = CGF.LambdaThisCaptureField;
 416     CGF.LambdaThisCaptureField = nullptr;
 417     BlockInfo = CGF.BlockInfo;
 418     CGF.BlockInfo = nullptr;
 419   }
 420
 421   ~InlinedOpenMPRegionRAII() {
 422     // Restore original CapturedStmtInfo only if we're done with code emission.
 423     auto *OldCSI =
 424         cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
 425     delete CGF.CapturedStmtInfo;
 426     CGF.CapturedStmtInfo = OldCSI;
 427     std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields);
 428     CGF.LambdaThisCaptureField = LambdaThisCaptureField;
 429     CGF.BlockInfo = BlockInfo;
 430   }
 431 };
 432
 433 /// Values for bit flags used in the ident_t to describe the fields.
 434 /// All enumeric elements are named and described in accordance with the code
 435 /// from http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
 436 enum OpenMPLocationFlags : unsigned {
 437   /// Use trampoline for internal microtask.
 438   OMP_IDENT_IMD = 0x01,
 439   /// Use c-style ident structure.
 440   OMP_IDENT_KMPC = 0x02,
 441   /// Atomic reduction option for kmpc_reduce.
 442   OMP_ATOMIC_REDUCE = 0x10,
 443   /// Explicit 'barrier' directive.
 444   OMP_IDENT_BARRIER_EXPL = 0x20,
 445   /// Implicit barrier in code.
 446   OMP_IDENT_BARRIER_IMPL = 0x40,
 447   /// Implicit barrier in 'for' directive.
 448   OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
 449   /// Implicit barrier in 'sections' directive.
 450   OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
 451   /// Implicit barrier in 'single' directive.
 452   OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
 453   /// Call of __kmp_for_static_init for static loop.
 454   OMP_IDENT_WORK_LOOP = 0x200,
 455   /// Call of __kmp_for_static_init for sections.
 456   OMP_IDENT_WORK_SECTIONS = 0x400,
 457   /// Call of __kmp_for_static_init for distribute.
 458   OMP_IDENT_WORK_DISTRIBUTE = 0x800,
 459   LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
 460 };
 461
 462 /// Describes ident structure that describes a source location.
 463 /// All descriptions are taken from
 464 /// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h
 465 /// Original structure:
 466 /// typedef struct ident {
 467 ///    kmp_int32 reserved_1;   /**<  might be used in Fortran;
 468 ///                                  see above  */
 469 ///    kmp_int32 flags;        /**<  also f.flags; KMP_IDENT_xxx flags;
 470 ///                                  KMP_IDENT_KMPC identifies this union
 471 ///                                  member  */
 472 ///    kmp_int32 reserved_2;   /**<  not really used in Fortran any more;
 473 ///                                  see above */
 474 ///#if USE_ITT_BUILD
 475 ///                            /*  but currently used for storing
 476 ///                                region-specific ITT */
 477 ///                            /*  contextual information. */
 478 ///#endif /* USE_ITT_BUILD */
 479 ///    kmp_int32 reserved_3;   /**< source[4] in Fortran, do not use for
 480 ///                                 C++  */
 481 ///    char const *psource;    /**< String describing the source location.
 482 ///                            The string is composed of semi-colon separated
 483 //                             fields which describe the source file,
 484 ///                            the function and a pair of line numbers that
 485 ///                            delimit the construct.
 486 ///                             */
 487 /// } ident_t;
 488 enum IdentFieldIndex {
 489   /// might be used in Fortran
 490   IdentField_Reserved_1,
 491   /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
 492   IdentField_Flags,
 493   /// Not really used in Fortran any more
 494   IdentField_Reserved_2,
 495   /// Source[4] in Fortran, do not use for C++
 496   IdentField_Reserved_3,
 497   /// String describing the source location. The string is composed of
 498   /// semi-colon separated fields which describe the source file, the function
 499   /// and a pair of line numbers that delimit the construct.
 500   IdentField_PSource
 501 };
 502
 503 /// Schedule types for 'omp for' loops (these enumerators are taken from
 504 /// the enum sched_type in kmp.h).
 505 enum OpenMPSchedType {
 506   /// Lower bound for default (unordered) versions.
 507   OMP_sch_lower = 32,
 508   OMP_sch_static_chunked = 33,
 509   OMP_sch_static = 34,
 510   OMP_sch_dynamic_chunked = 35,
 511   OMP_sch_guided_chunked = 36,
 512   OMP_sch_runtime = 37,
 513   OMP_sch_auto = 38,
 514   /// static with chunk adjustment (e.g., simd)
 515   OMP_sch_static_balanced_chunked = 45,
 516   /// Lower bound for 'ordered' versions.
 517   OMP_ord_lower = 64,
 518   OMP_ord_static_chunked = 65,
 519   OMP_ord_static = 66,
 520   OMP_ord_dynamic_chunked = 67,
 521   OMP_ord_guided_chunked = 68,
 522   OMP_ord_runtime = 69,
 523   OMP_ord_auto = 70,
 524   OMP_sch_default = OMP_sch_static,
 525   /// dist_schedule types
 526   OMP_dist_sch_static_chunked = 91,
 527   OMP_dist_sch_static = 92,
 528   /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
 529   /// Set if the monotonic schedule modifier was present.
 530   OMP_sch_modifier_monotonic = (1 << 29),
 531   /// Set if the nonmonotonic schedule modifier was present.
 532   OMP_sch_modifier_nonmonotonic = (1 << 30),
 533 };
 534
 535 enum OpenMPRTLFunction {
 536   /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc,
 537   /// kmpc_micro microtask, ...);
 538   OMPRTL__kmpc_fork_call,
 539   /// Call to void *__kmpc_threadprivate_cached(ident_t *loc,
 540   /// kmp_int32 global_tid, void *data, size_t size, void ***cache);
 541   OMPRTL__kmpc_threadprivate_cached,
 542   /// Call to void __kmpc_threadprivate_register( ident_t *,
 543   /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
 544   OMPRTL__kmpc_threadprivate_register,
 545   // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc);
 546   OMPRTL__kmpc_global_thread_num,
 547   // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
 548   // kmp_critical_name *crit);
 549   OMPRTL__kmpc_critical,
 550   // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32
 551   // global_tid, kmp_critical_name *crit, uintptr_t hint);
 552   OMPRTL__kmpc_critical_with_hint,
 553   // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
 554   // kmp_critical_name *crit);
 555   OMPRTL__kmpc_end_critical,
 556   // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
 557   // global_tid);
 558   OMPRTL__kmpc_cancel_barrier,
 559   // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
 560   OMPRTL__kmpc_barrier,
 561   // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
 562   OMPRTL__kmpc_for_static_fini,
 563   // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
 564   // global_tid);
 565   OMPRTL__kmpc_serialized_parallel,
 566   // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
 567   // global_tid);
 568   OMPRTL__kmpc_end_serialized_parallel,
 569   // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
 570   // kmp_int32 num_threads);
 571   OMPRTL__kmpc_push_num_threads,
 572   // Call to void __kmpc_flush(ident_t *loc);
 573   OMPRTL__kmpc_flush,
 574   // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);
 575   OMPRTL__kmpc_master,
 576   // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid);
 577   OMPRTL__kmpc_end_master,
 578   // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
 579   // int end_part);
 580   OMPRTL__kmpc_omp_taskyield,
 581   // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);
 582   OMPRTL__kmpc_single,
 583   // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid);
 584   OMPRTL__kmpc_end_single,
 585   // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
 586   // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
 587   // kmp_routine_entry_t *task_entry);
 588   OMPRTL__kmpc_omp_task_alloc,
 589   // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *
 590   // new_task);
 591   OMPRTL__kmpc_omp_task,
 592   // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
 593   // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
 594   // kmp_int32 didit);
 595   OMPRTL__kmpc_copyprivate,
 596   // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
 597   // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
 598   // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
 599   OMPRTL__kmpc_reduce,
 600   // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
 601   // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
 602   // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
 603   // *lck);
 604   OMPRTL__kmpc_reduce_nowait,
 605   // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
 606   // kmp_critical_name *lck);
 607   OMPRTL__kmpc_end_reduce,
 608   // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
 609   // kmp_critical_name *lck);
 610   OMPRTL__kmpc_end_reduce_nowait,
 611   // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
 612   // kmp_task_t * new_task);
 613   OMPRTL__kmpc_omp_task_begin_if0,
 614   // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
 615   // kmp_task_t * new_task);
 616   OMPRTL__kmpc_omp_task_complete_if0,
 617   // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
 618   OMPRTL__kmpc_ordered,
 619   // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
 620   OMPRTL__kmpc_end_ordered,
 621   // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
 622   // global_tid);
 623   OMPRTL__kmpc_omp_taskwait,
 624   // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
 625   OMPRTL__kmpc_taskgroup,
 626   // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
 627   OMPRTL__kmpc_end_taskgroup,
 628   // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
 629   // int proc_bind);
 630   OMPRTL__kmpc_push_proc_bind,
 631   // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32
 632   // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t
 633   // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
 634   OMPRTL__kmpc_omp_task_with_deps,
 635   // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32
 636   // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
 637   // ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
 638   OMPRTL__kmpc_omp_wait_deps,
 639   // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
 640   // global_tid, kmp_int32 cncl_kind);
 641   OMPRTL__kmpc_cancellationpoint,
 642   // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
 643   // kmp_int32 cncl_kind);
 644   OMPRTL__kmpc_cancel,
 645   // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
 646   // kmp_int32 num_teams, kmp_int32 thread_limit);
 647   OMPRTL__kmpc_push_num_teams,
 648   // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
 649   // microtask, ...);
 650   OMPRTL__kmpc_fork_teams,
 651   // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
 652   // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
 653   // sched, kmp_uint64 grainsize, void *task_dup);
 654   OMPRTL__kmpc_taskloop,
 655   // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
 656   // num_dims, struct kmp_dim *dims);
 657   OMPRTL__kmpc_doacross_init,
 658   // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
 659   OMPRTL__kmpc_doacross_fini,
 660   // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
 661   // *vec);
 662   OMPRTL__kmpc_doacross_post,
 663   // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
 664   // *vec);
 665   OMPRTL__kmpc_doacross_wait,
 666   // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void
 667   // *data);
 668   OMPRTL__kmpc_task_reduction_init,
 669   // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
 670   // *d);
 671   OMPRTL__kmpc_task_reduction_get_th_data,
 672
 673   //
 674   // Offloading related calls
 675   //
 676   // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
 677   // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
 678   // *arg_types);
 679   OMPRTL__tgt_target,
 680   // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
 681   // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
 682   // *arg_types);
 683   OMPRTL__tgt_target_nowait,
 684   // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
 685   // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
 686   // *arg_types, int32_t num_teams, int32_t thread_limit);
 687   OMPRTL__tgt_target_teams,
 688   // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void
 689   // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
 690   // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
 691   OMPRTL__tgt_target_teams_nowait,
 692   // Call to void __tgt_register_lib(__tgt_bin_desc *desc);
 693   OMPRTL__tgt_register_lib,
 694   // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc);
 695   OMPRTL__tgt_unregister_lib,
 696   // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
 697   // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
 698   OMPRTL__tgt_target_data_begin,
 699   // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
 700   // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
 701   // *arg_types);
 702   OMPRTL__tgt_target_data_begin_nowait,
 703   // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
 704   // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
 705   OMPRTL__tgt_target_data_end,
 706   // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t
 707   // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
 708   // *arg_types);
 709   OMPRTL__tgt_target_data_end_nowait,
 710   // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
 711   // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
 712   OMPRTL__tgt_target_data_update,
 713   // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t
 714   // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
 715   // *arg_types);
 716   OMPRTL__tgt_target_data_update_nowait,
 717 };
 718
 719 /// A basic class for pre|post-action for advanced codegen sequence for OpenMP
 720 /// region.
 721 class CleanupTy final : public EHScopeStack::Cleanup {
 722   PrePostActionTy *Action;
 723
 724 public:
 725   explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
 726   void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
 727     if (!CGF.HaveInsertPoint())
 728       return;
 729     Action->Exit(CGF);
 730   }
 731 };
 732
 733 } // anonymous namespace
 734
 735 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
 736   CodeGenFunction::RunCleanupsScope Scope(CGF);
 737   if (PrePostAction) {
 738     CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction);
 739     Callback(CodeGen, CGF, *PrePostAction);
 740   } else {
 741     PrePostActionTy Action;
 742     Callback(CodeGen, CGF, Action);
 743   }
 744 }
 745
 746 /// Check if the combiner is a call to UDR combiner and if it is so return the
 747 /// UDR decl used for reduction.
 748 static const OMPDeclareReductionDecl *
 749 getReductionInit(const Expr *ReductionOp) {
 750   if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
 751     if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
 752       if (const auto *DRE =
 753               dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
 754         if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
 755           return DRD;
 756   return nullptr;
 757 }
 758
 759 static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
 760                                              const OMPDeclareReductionDecl *DRD,
 761                                              const Expr *InitOp,
 762                                              Address Private, Address Original,
 763                                              QualType Ty) {
 764   if (DRD->getInitializer()) {
 765     std::pair<llvm::Function *, llvm::Function *> Reduction =
 766         CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
 767     const auto *CE = cast<CallExpr>(InitOp);
 768     const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
 769     const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
 770     const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
 771     const auto *LHSDRE =
 772         cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
 773     const auto *RHSDRE =
 774         cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
 775     CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
 776     PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
 777                             [=]() { return Private; });
 778     PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
 779                             [=]() { return Original; });
 780     (void)PrivateScope.Privatize();
 781     RValue Func = RValue::get(Reduction.second);
 782     CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
 783     CGF.EmitIgnoredExpr(InitOp);
 784   } else {
 785     llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
 786     std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"});
 787     auto *GV = new llvm::GlobalVariable(
 788         CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
 789         llvm::GlobalValue::PrivateLinkage, Init, Name);
 790     LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
 791     RValue InitRVal;
 792     switch (CGF.getEvaluationKind(Ty)) {
 793     case TEK_Scalar:
 794       InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation());
 795       break;
 796     case TEK_Complex:
 797       InitRVal =
 798           RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation()));
 799       break;
 800     case TEK_Aggregate:
 801       InitRVal = RValue::getAggregate(LV.getAddress());
 802       break;
 803     }
 804     OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue);
 805     CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
 806     CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
 807                          /*IsInitializer=*/false);
 808   }
 809 }
 810
 811 /// Emit initialization of arrays of complex types.
 812 /// \param DestAddr Address of the array.
 813 /// \param Type Type of array.
 814 /// \param Init Initial expression of array.
 815 /// \param SrcAddr Address of the original array.
 816 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
 817                                  QualType Type, bool EmitDeclareReductionInit,
 818                                  const Expr *Init,
 819                                  const OMPDeclareReductionDecl *DRD,
 820                                  Address SrcAddr = Address::invalid()) {
 821   // Perform element-by-element initialization.
 822   QualType ElementTy;
 823
 824   // Drill down to the base element type on both arrays.
 825   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
 826   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
 827   DestAddr =
 828       CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
 829   if (DRD)
 830     SrcAddr =
 831         CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
 832
 833   llvm::Value *SrcBegin = nullptr;
 834   if (DRD)
 835     SrcBegin = SrcAddr.getPointer();
 836   llvm::Value *DestBegin = DestAddr.getPointer();
 837   // Cast from pointer to array type to pointer to single element.
 838   llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
 839   // The basic structure here is a while-do loop.
 840   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
 841   llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
 842   llvm::Value *IsEmpty =
 843       CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
 844   CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
 845
 846   // Enter the loop body, making that address the current address.
 847   llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
 848   CGF.EmitBlock(BodyBB);
 849
 850   CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
 851
 852   llvm::PHINode *SrcElementPHI = nullptr;
 853   Address SrcElementCurrent = Address::invalid();
 854   if (DRD) {
 855     SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
 856                                           "omp.arraycpy.srcElementPast");
 857     SrcElementPHI->addIncoming(SrcBegin, EntryBB);
 858     SrcElementCurrent =
 859         Address(SrcElementPHI,
 860                 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
 861   }
 862   llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
 863       DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
 864   DestElementPHI->addIncoming(DestBegin, EntryBB);
 865   Address DestElementCurrent =
 866       Address(DestElementPHI,
 867               DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
 868
 869   // Emit copy.
 870   {
 871     CodeGenFunction::RunCleanupsScope InitScope(CGF);
 872     if (EmitDeclareReductionInit) {
 873       emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
 874                                        SrcElementCurrent, ElementTy);
 875     } else
 876       CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
 877                            /*IsInitializer=*/false);
 878   }
 879
 880   if (DRD) {
 881     // Shift the address forward by one element.
 882     llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
 883         SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
 884     SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
 885   }
 886
 887   // Shift the address forward by one element.
 888   llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
 889       DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
 890   // Check whether we've reached the end.
 891   llvm::Value *Done =
 892       CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
 893   CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
 894   DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
 895
 896   // Done.
 897   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
 898 }
 899
 900 static llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy>
 901 isDeclareTargetDeclaration(const ValueDecl *VD) {
 902   for (const Decl *D : VD->redecls()) {
 903     if (!D->hasAttrs())
 904       continue;
 905     if (const auto *Attr = D->getAttr<OMPDeclareTargetDeclAttr>())
 906       return Attr->getMapType();
 907   }
 908   if (const auto *V = dyn_cast<VarDecl>(VD)) {
 909     if (const VarDecl *TD = V->getTemplateInstantiationPattern())
 910       return isDeclareTargetDeclaration(TD);
 911   } else if (const auto *FD = dyn_cast<FunctionDecl>(VD)) {
 912     if (const auto *TD = FD->getTemplateInstantiationPattern())
 913       return isDeclareTargetDeclaration(TD);
 914   }
 915
 916   return llvm::None;
 917 }
 918
 919 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
 920   return CGF.EmitOMPSharedLValue(E);
 921 }
 922
 923 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
 924                                             const Expr *E) {
 925   if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
 926     return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
 927   return LValue();
 928 }
 929
 930 void ReductionCodeGen::emitAggregateInitialization(
 931     CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
 932     const OMPDeclareReductionDecl *DRD) {
 933   // Emit VarDecl with copy init for arrays.
 934   // Get the address of the original variable captured in current
 935   // captured region.
 936   const auto *PrivateVD =
 937       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
 938   bool EmitDeclareReductionInit =
 939       DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
 940   EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(),
 941                        EmitDeclareReductionInit,
 942                        EmitDeclareReductionInit ? ClausesData[N].ReductionOp
 943                                                 : PrivateVD->getInit(),
 944                        DRD, SharedLVal.getAddress());
 945 }
 946
 947 ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
 948                                    ArrayRef<const Expr *> Privates,
 949                                    ArrayRef<const Expr *> ReductionOps) {
 950   ClausesData.reserve(Shareds.size());
 951   SharedAddresses.reserve(Shareds.size());
 952   Sizes.reserve(Shareds.size());
 953   BaseDecls.reserve(Shareds.size());
 954   auto IPriv = Privates.begin();
 955   auto IRed = ReductionOps.begin();
 956   for (const Expr *Ref : Shareds) {
 957     ClausesData.emplace_back(Ref, *IPriv, *IRed);
 958     std::advance(IPriv, 1);
 959     std::advance(IRed, 1);
 960   }
 961 }
 962
 963 void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) {
 964   assert(SharedAddresses.size() == N &&
 965          "Number of generated lvalues must be exactly N.");
 966   LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
 967   LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
 968   SharedAddresses.emplace_back(First, Second);
 969 }
 970
 971 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
 972   const auto *PrivateVD =
 973       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
 974   QualType PrivateType = PrivateVD->getType();
 975   bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
 976   if (!PrivateType->isVariablyModifiedType()) {
 977     Sizes.emplace_back(
 978         CGF.getTypeSize(
 979             SharedAddresses[N].first.getType().getNonReferenceType()),
 980         nullptr);
 981     return;
 982   }
 983   llvm::Value *Size;
 984   llvm::Value *SizeInChars;
 985   auto *ElemType =
 986       cast<llvm::PointerType>(SharedAddresses[N].first.getPointer()->getType())
 987           ->getElementType();
 988   auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
 989   if (AsArraySection) {
 990     Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(),
 991                                      SharedAddresses[N].first.getPointer());
 992     Size = CGF.Builder.CreateNUWAdd(
 993         Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
 994     SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf);
 995   } else {
 996     SizeInChars = CGF.getTypeSize(
 997         SharedAddresses[N].first.getType().getNonReferenceType());
 998     Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
 999   }
1000   Sizes.emplace_back(SizeInChars, Size);
1001   CodeGenFunction::OpaqueValueMapping OpaqueMap(
1002       CGF,
1003       cast<OpaqueValueExpr>(
1004           CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1005       RValue::get(Size));
1006   CGF.EmitVariablyModifiedType(PrivateType);
1007 }
1008
1009 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
1010                                          llvm::Value *Size) {
1011   const auto *PrivateVD =
1012       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1013   QualType PrivateType = PrivateVD->getType();
1014   if (!PrivateType->isVariablyModifiedType()) {
1015     assert(!Size && !Sizes[N].second &&
1016            "Size should be nullptr for non-variably modified reduction "
1017            "items.");
1018     return;
1019   }
1020   CodeGenFunction::OpaqueValueMapping OpaqueMap(
1021       CGF,
1022       cast<OpaqueValueExpr>(
1023           CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()),
1024       RValue::get(Size));
1025   CGF.EmitVariablyModifiedType(PrivateType);
1026 }
1027
1028 void ReductionCodeGen::emitInitialization(
1029     CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal,
1030     llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
1031   assert(SharedAddresses.size() > N && "No variable was generated");
1032   const auto *PrivateVD =
1033       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1034   const OMPDeclareReductionDecl *DRD =
1035       getReductionInit(ClausesData[N].ReductionOp);
1036   QualType PrivateType = PrivateVD->getType();
1037   PrivateAddr = CGF.Builder.CreateElementBitCast(
1038       PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1039   QualType SharedType = SharedAddresses[N].first.getType();
1040   SharedLVal = CGF.MakeAddrLValue(
1041       CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(),
1042                                        CGF.ConvertTypeForMem(SharedType)),
1043       SharedType, SharedAddresses[N].first.getBaseInfo(),
1044       CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType));
1045   if (CGF.getContext().getAsArrayType(PrivateVD->getType())) {
1046     emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1047   } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1048     emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp,
1049                                      PrivateAddr, SharedLVal.getAddress(),
1050                                      SharedLVal.getType());
1051   } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1052              !CGF.isTrivialInitializer(PrivateVD->getInit())) {
1053     CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr,
1054                          PrivateVD->getType().getQualifiers(),
1055                          /*IsInitializer=*/false);
1056   }
1057 }
1058
1059 bool ReductionCodeGen::needCleanups(unsigned N) {
1060   const auto *PrivateVD =
1061       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1062   QualType PrivateType = PrivateVD->getType();
1063   QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1064   return DTorKind != QualType::DK_none;
1065 }
1066
1067 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
1068                                     Address PrivateAddr) {
1069   const auto *PrivateVD =
1070       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1071   QualType PrivateType = PrivateVD->getType();
1072   QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
1073   if (needCleanups(N)) {
1074     PrivateAddr = CGF.Builder.CreateElementBitCast(
1075         PrivateAddr, CGF.ConvertTypeForMem(PrivateType));
1076     CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType);
1077   }
1078 }
1079
1080 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1081                           LValue BaseLV) {
1082   BaseTy = BaseTy.getNonReferenceType();
1083   while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1084          !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1085     if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
1086       BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy);
1087     } else {
1088       LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy);
1089       BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
1090     }
1091     BaseTy = BaseTy->getPointeeType();
1092   }
1093   return CGF.MakeAddrLValue(
1094       CGF.Builder.CreateElementBitCast(BaseLV.getAddress(),
1095                                        CGF.ConvertTypeForMem(ElTy)),
1096       BaseLV.getType(), BaseLV.getBaseInfo(),
1097       CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType()));
1098 }
1099
1100 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
1101                           llvm::Type *BaseLVType, CharUnits BaseLVAlignment,
1102                           llvm::Value *Addr) {
1103   Address Tmp = Address::invalid();
1104   Address TopTmp = Address::invalid();
1105   Address MostTopTmp = Address::invalid();
1106   BaseTy = BaseTy.getNonReferenceType();
1107   while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
1108          !CGF.getContext().hasSameType(BaseTy, ElTy)) {
1109     Tmp = CGF.CreateMemTemp(BaseTy);
1110     if (TopTmp.isValid())
1111       CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp);
1112     else
1113       MostTopTmp = Tmp;
1114     TopTmp = Tmp;
1115     BaseTy = BaseTy->getPointeeType();
1116   }
1117   llvm::Type *Ty = BaseLVType;
1118   if (Tmp.isValid())
1119     Ty = Tmp.getElementType();
1120   Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty);
1121   if (Tmp.isValid()) {
1122     CGF.Builder.CreateStore(Addr, Tmp);
1123     return MostTopTmp;
1124   }
1125   return Address(Addr, BaseLVAlignment);
1126 }
1127
1128 static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
1129   const VarDecl *OrigVD = nullptr;
1130   if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1131     const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
1132     while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1133       Base = TempOASE->getBase()->IgnoreParenImpCasts();
1134     while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1135       Base = TempASE->getBase()->IgnoreParenImpCasts();
1136     DE = cast<DeclRefExpr>(Base);
1137     OrigVD = cast<VarDecl>(DE->getDecl());
1138   } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1139     const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
1140     while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1141       Base = TempASE->getBase()->IgnoreParenImpCasts();
1142     DE = cast<DeclRefExpr>(Base);
1143     OrigVD = cast<VarDecl>(DE->getDecl());
1144   }
1145   return OrigVD;
1146 }
1147
1148 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
1149                                                Address PrivateAddr) {
1150   const DeclRefExpr *DE;
1151   if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) {
1152     BaseDecls.emplace_back(OrigVD);
1153     LValue OriginalBaseLValue = CGF.EmitLValue(DE);
1154     LValue BaseLValue =
1155         loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1156                     OriginalBaseLValue);
1157     llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
1158         BaseLValue.getPointer(), SharedAddresses[N].first.getPointer());
1159     llvm::Value *PrivatePointer =
1160         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1161             PrivateAddr.getPointer(),
1162             SharedAddresses[N].first.getAddress().getType());
1163     llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment);
1164     return castToBase(CGF, OrigVD->getType(),
1165                       SharedAddresses[N].first.getType(),
1166                       OriginalBaseLValue.getAddress().getType(),
1167                       OriginalBaseLValue.getAlignment(), Ptr);
1168   }
1169   BaseDecls.emplace_back(
1170       cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1171   return PrivateAddr;
1172 }
1173
1174 bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
1175   const OMPDeclareReductionDecl *DRD =
1176       getReductionInit(ClausesData[N].ReductionOp);
1177   return DRD && DRD->getInitializer();
1178 }
1179
1180 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1181   return CGF.EmitLoadOfPointerLValue(
1182       CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1183       getThreadIDVariable()->getType()->castAs<PointerType>());
1184 }
1185
1186 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
1187   if (!CGF.HaveInsertPoint())
1188     return;
1189   // 1.2.2 OpenMP Language Terminology
1190   // Structured block - An executable statement with a single entry at the
1191   // top and a single exit at the bottom.
1192   // The point of exit cannot be a branch out of the structured block.
1193   // longjmp() and throw() must not violate the entry/exit criteria.
1194   CGF.EHStack.pushTerminate();
1195   CodeGen(CGF);
1196   CGF.EHStack.popTerminate();
1197 }
1198
1199 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1200     CodeGenFunction &CGF) {
1201   return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()),
1202                             getThreadIDVariable()->getType(),
1203                             AlignmentSource::Decl);
1204 }
1205
1206 static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1207                                        QualType FieldTy) {
1208   auto *Field = FieldDecl::Create(
1209       C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1210       C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()),
1211       /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1212   Field->setAccess(AS_public);
1213   DC->addDecl(Field);
1214   return Field;
1215 }
1216
1217 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
1218                                  StringRef Separator)
1219     : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1220       OffloadEntriesInfoManager(CGM) {
1221   ASTContext &C = CGM.getContext();
1222   RecordDecl *RD = C.buildImplicitRecord("ident_t");
1223   QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1224   RD->startDefinition();
1225   // reserved_1
1226   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1227   // flags
1228   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1229   // reserved_2
1230   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1231   // reserved_3
1232   addFieldToRecordDecl(C, RD, KmpInt32Ty);
1233   // psource
1234   addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1235   RD->completeDefinition();
1236   IdentQTy = C.getRecordType(RD);
1237   IdentTy = CGM.getTypes().ConvertRecordDeclType(RD);
1238   KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
1239
1240   loadOffloadInfoMetadata();
1241 }
1242
1243 void CGOpenMPRuntime::clear() {
1244   InternalVars.clear();
1245 }
1246
1247 std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1248   SmallString<128> Buffer;
1249   llvm::raw_svector_ostream OS(Buffer);
1250   StringRef Sep = FirstSeparator;
1251   for (StringRef Part : Parts) {
1252     OS << Sep << Part;
1253     Sep = Separator;
1254   }
1255   return OS.str();
1256 }
1257
1258 static llvm::Function *
1259 emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1260                           const Expr *CombinerInitializer, const VarDecl *In,
1261                           const VarDecl *Out, bool IsCombiner) {
1262   // void .omp_combiner.(Ty *in, Ty *out);
1263   ASTContext &C = CGM.getContext();
1264   QualType PtrTy = C.getPointerType(Ty).withRestrict();
1265   FunctionArgList Args;
1266   ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
1267                                /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1268   ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
1269                               /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other);
1270   Args.push_back(&OmpOutParm);
1271   Args.push_back(&OmpInParm);
1272   const CGFunctionInfo &FnInfo =
1273       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
1274   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
1275   std::string Name = CGM.getOpenMPRuntime().getName(
1276       {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1277   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
1278                                     Name, &CGM.getModule());
1279   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
1280   Fn->removeFnAttr(llvm::Attribute::NoInline);
1281   Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1282   Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1283   CodeGenFunction CGF(CGM);
1284   // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1285   // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1286   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(),
1287                     Out->getLocation());
1288   CodeGenFunction::OMPPrivateScope Scope(CGF);
1289   Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm);
1290   Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1291     return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>())
1292         .getAddress();
1293   });
1294   Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm);
1295   Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1296     return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>())
1297         .getAddress();
1298   });
1299   (void)Scope.Privatize();
1300   if (!IsCombiner && Out->hasInit() &&
1301       !CGF.isTrivialInitializer(Out->getInit())) {
1302     CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out),
1303                          Out->getType().getQualifiers(),
1304                          /*IsInitializer=*/true);
1305   }
1306   if (CombinerInitializer)
1307     CGF.EmitIgnoredExpr(CombinerInitializer);
1308   Scope.ForceCleanup();
1309   CGF.FinishFunction();
1310   return Fn;
1311 }
1312
1313 void CGOpenMPRuntime::emitUserDefinedReduction(
1314     CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1315   if (UDRMap.count(D) > 0)
1316     return;
1317   ASTContext &C = CGM.getContext();
1318   if (!In || !Out) {
1319     In = &C.Idents.get("omp_in");
1320     Out = &C.Idents.get("omp_out");
1321   }
1322   llvm::Function *Combiner = emitCombinerOrInitializer(
1323       CGM, D->getType(), D->getCombiner(), cast<VarDecl>(D->lookup(In).front()),
1324       cast<VarDecl>(D->lookup(Out).front()),
1325       /*IsCombiner=*/true);
1326   llvm::Function *Initializer = nullptr;
1327   if (const Expr *Init = D->getInitializer()) {
1328     if (!Priv || !Orig) {
1329       Priv = &C.Idents.get("omp_priv");
1330       Orig = &C.Idents.get("omp_orig");
1331     }
1332     Initializer = emitCombinerOrInitializer(
1333         CGM, D->getType(),
1334         D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init
1335                                                                      : nullptr,
1336         cast<VarDecl>(D->lookup(Orig).front()),
1337         cast<VarDecl>(D->lookup(Priv).front()),
1338         /*IsCombiner=*/false);
1339   }
1340   UDRMap.try_emplace(D, Combiner, Initializer);
1341   if (CGF) {
1342     auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn);
1343     Decls.second.push_back(D);
1344   }
1345 }
1346
1347 std::pair<llvm::Function *, llvm::Function *>
1348 CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1349   auto I = UDRMap.find(D);
1350   if (I != UDRMap.end())
1351     return I->second;
1352   emitUserDefinedReduction(/*CGF=*/nullptr, D);
1353   return UDRMap.lookup(D);
1354 }
1355
1356 static llvm::Value *emitParallelOrTeamsOutlinedFunction(
1357     CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1358     const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1359     const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1360   assert(ThreadIDVar->getType()->isPointerType() &&
1361          "thread id variable must be of type kmp_int32 *");
1362   CodeGenFunction CGF(CGM, true);
1363   bool HasCancel = false;
1364   if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1365     HasCancel = OPD->hasCancel();
1366   else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1367     HasCancel = OPSD->hasCancel();
1368   else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1369     HasCancel = OPFD->hasCancel();
1370   else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1371     HasCancel = OPFD->hasCancel();
1372   else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1373     HasCancel = OPFD->hasCancel();
1374   else if (const auto *OPFD =
1375                dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1376     HasCancel = OPFD->hasCancel();
1377   else if (const auto *OPFD =
1378                dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1379     HasCancel = OPFD->hasCancel();
1380   CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1381                                     HasCancel, OutlinedHelperName);
1382   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1383   return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1384 }
1385
1386 llvm::Value *CGOpenMPRuntime::emitParallelOutlinedFunction(
1387     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1388     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1389   const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel);
1390   return emitParallelOrTeamsOutlinedFunction(
1391       CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1392 }
1393
1394 llvm::Value *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1395     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1396     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1397   const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams);
1398   return emitParallelOrTeamsOutlinedFunction(
1399       CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen);
1400 }
1401
1402 llvm::Value *CGOpenMPRuntime::emitTaskOutlinedFunction(
1403     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1404     const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1405     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1406     bool Tied, unsigned &NumberOfParts) {
1407   auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1408                                               PrePostActionTy &) {
1409     llvm::Value *ThreadID = getThreadID(CGF, D.getLocStart());
1410     llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getLocStart());
1411     llvm::Value *TaskArgs[] = {
1412         UpLoc, ThreadID,
1413         CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1414                                     TaskTVar->getType()->castAs<PointerType>())
1415             .getPointer()};
1416     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs);
1417   };
1418   CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1419                                                             UntiedCodeGen);
1420   CodeGen.setAction(Action);
1421   assert(!ThreadIDVar->getType()->isPointerType() &&
1422          "thread id variable must be of type kmp_int32 for tasks");
1423   const OpenMPDirectiveKind Region =
1424       isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
1425                                                       : OMPD_task;
1426   const CapturedStmt *CS = D.getCapturedStmt(Region);
1427   const auto *TD = dyn_cast<OMPTaskDirective>(&D);
1428   CodeGenFunction CGF(CGM, true);
1429   CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1430                                         InnermostKind,
1431                                         TD ? TD->hasCancel() : false, Action);
1432   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1433   llvm::Value *Res = CGF.GenerateCapturedStmtFunction(*CS);
1434   if (!Tied)
1435     NumberOfParts = Action.getNumberOfParts();
1436   return Res;
1437 }
1438
1439 static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM,
1440                              const RecordDecl *RD, const CGRecordLayout &RL,
1441                              ArrayRef<llvm::Constant *> Data) {
1442   llvm::StructType *StructTy = RL.getLLVMType();
1443   unsigned PrevIdx = 0;
1444   ConstantInitBuilder CIBuilder(CGM);
1445   auto DI = Data.begin();
1446   for (const FieldDecl *FD : RD->fields()) {
1447     unsigned Idx = RL.getLLVMFieldNo(FD);
1448     // Fill the alignment.
1449     for (unsigned I = PrevIdx; I < Idx; ++I)
1450       Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1451     PrevIdx = Idx + 1;
1452     Fields.add(*DI);
1453     ++DI;
1454   }
1455 }
1456
1457 template <class... As>
1458 static llvm::GlobalVariable *
1459 createConstantGlobalStruct(CodeGenModule &CGM, QualType Ty,
1460                            ArrayRef<llvm::Constant *> Data, const Twine &Name,
1461                            As &&... Args) {
1462   const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1463   const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1464   ConstantInitBuilder CIBuilder(CGM);
1465   ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType());
1466   buildStructValue(Fields, CGM, RD, RL, Data);
1467   return Fields.finishAndCreateGlobal(
1468       Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty),
1469       /*isConstant=*/true, std::forward<As>(Args)...);
1470 }
1471
1472 template <typename T>
1473 static void
1474 createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty,
1475                                          ArrayRef<llvm::Constant *> Data,
1476                                          T &Parent) {
1477   const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl());
1478   const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD);
1479   ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType());
1480   buildStructValue(Fields, CGM, RD, RL, Data);
1481   Fields.finishAndAddTo(Parent);
1482 }
1483
1484 Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) {
1485   CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1486   llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
1487   if (!Entry) {
1488     if (!DefaultOpenMPPSource) {
1489       // Initialize default location for psource field of ident_t structure of
1490       // all ident_t objects. Format is ";file;function;line;column;;".
1491       // Taken from
1492       // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
1493       DefaultOpenMPPSource =
1494           CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer();
1495       DefaultOpenMPPSource =
1496           llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
1497     }
1498
1499     llvm::Constant *Data[] = {llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1500                               llvm::ConstantInt::get(CGM.Int32Ty, Flags),
1501                               llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1502                               llvm::ConstantInt::getNullValue(CGM.Int32Ty),
1503                               DefaultOpenMPPSource};
1504     llvm::GlobalValue *DefaultOpenMPLocation = createConstantGlobalStruct(
1505         CGM, IdentQTy, Data, "", llvm::GlobalValue::PrivateLinkage);
1506     DefaultOpenMPLocation->setUnnamedAddr(
1507         llvm::GlobalValue::UnnamedAddr::Global);
1508
1509     OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation;
1510   }
1511   return Address(Entry, Align);
1512 }
1513
1514 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1515                                                  SourceLocation Loc,
1516                                                  unsigned Flags) {
1517   Flags |= OMP_IDENT_KMPC;
1518   // If no debug info is generated - return global default location.
1519   if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo ||
1520       Loc.isInvalid())
1521     return getOrCreateDefaultLocation(Flags).getPointer();
1522
1523   assert(CGF.CurFn && "No function in current CodeGenFunction.");
1524
1525   CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy);
1526   Address LocValue = Address::invalid();
1527   auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1528   if (I != OpenMPLocThreadIDMap.end())
1529     LocValue = Address(I->second.DebugLoc, Align);
1530
1531   // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
1532   // GetOpenMPThreadID was called before this routine.
1533   if (!LocValue.isValid()) {
1534     // Generate "ident_t .kmpc_loc.addr;"
1535     Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr");
1536     auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1537     Elem.second.DebugLoc = AI.getPointer();
1538     LocValue = AI;
1539
1540     CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1541     CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
1542     CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
1543                              CGF.getTypeSize(IdentQTy));
1544   }
1545
1546   // char **psource = &.kmpc_loc_<flags>.addr.psource;
1547   LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy);
1548   auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1549   LValue PSource =
1550       CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource));
1551
1552   llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
1553   if (OMPDebugLoc == nullptr) {
1554     SmallString<128> Buffer2;
1555     llvm::raw_svector_ostream OS2(Buffer2);
1556     // Build debug location
1557     PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1558     OS2 << ";" << PLoc.getFilename() << ";";
1559     if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl))
1560       OS2 << FD->getQualifiedNameAsString();
1561     OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1562     OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
1563     OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
1564   }
1565   // *psource = ";<File>;<Function>;<Line>;<Column>;;";
1566   CGF.EmitStoreOfScalar(OMPDebugLoc, PSource);
1567
1568   // Our callers always pass this to a runtime function, so for
1569   // convenience, go ahead and return a naked pointer.
1570   return LocValue.getPointer();
1571 }
1572
1573 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1574                                           SourceLocation Loc) {
1575   assert(CGF.CurFn && "No function in current CodeGenFunction.");
1576
1577   llvm::Value *ThreadID = nullptr;
1578   // Check whether we've already cached a load of the thread id in this
1579   // function.
1580   auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
1581   if (I != OpenMPLocThreadIDMap.end()) {
1582     ThreadID = I->second.ThreadID;
1583     if (ThreadID != nullptr)
1584       return ThreadID;
1585   }
1586   // If exceptions are enabled, do not use parameter to avoid possible crash.
1587   if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1588       !CGF.getLangOpts().CXXExceptions ||
1589       CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1590     if (auto *OMPRegionInfo =
1591             dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1592       if (OMPRegionInfo->getThreadIDVariable()) {
1593         // Check if this an outlined function with thread id passed as argument.
1594         LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1595         ThreadID = CGF.EmitLoadOfScalar(LVal, Loc);
1596         // If value loaded in entry block, cache it and use it everywhere in
1597         // function.
1598         if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
1599           auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1600           Elem.second.ThreadID = ThreadID;
1601         }
1602         return ThreadID;
1603       }
1604     }
1605   }
1606
1607   // This is not an outlined function region - need to call __kmpc_int32
1608   // kmpc_global_thread_num(ident_t *loc).
1609   // Generate thread id value and cache this value for use across the
1610   // function.
1611   CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1612   CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
1613   llvm::CallInst *Call = CGF.Builder.CreateCall(
1614       createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
1615       emitUpdateLocation(CGF, Loc));
1616   Call->setCallingConv(CGF.getRuntimeCC());
1617   auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
1618   Elem.second.ThreadID = Call;
1619   return Call;
1620 }
1621
1622 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1623   assert(CGF.CurFn && "No function in current CodeGenFunction.");
1624   if (OpenMPLocThreadIDMap.count(CGF.CurFn))
1625     OpenMPLocThreadIDMap.erase(CGF.CurFn);
1626   if (FunctionUDRMap.count(CGF.CurFn) > 0) {
1627     for(auto *D : FunctionUDRMap[CGF.CurFn])
1628       UDRMap.erase(D);
1629     FunctionUDRMap.erase(CGF.CurFn);
1630   }
1631 }
1632
1633 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1634   return IdentTy->getPointerTo();
1635 }
1636
1637 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
1638   if (!Kmpc_MicroTy) {
1639     // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
1640     llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1641                                  llvm::PointerType::getUnqual(CGM.Int32Ty)};
1642     Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
1643   }
1644   return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1645 }
1646
1647 llvm::Constant *
1648 CGOpenMPRuntime::createRuntimeFunction(unsigned Function) {
1649   llvm::Constant *RTLFn = nullptr;
1650   switch (static_cast<OpenMPRTLFunction>(Function)) {
1651   case OMPRTL__kmpc_fork_call: {
1652     // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
1653     // microtask, ...);
1654     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1655                                 getKmpc_MicroPointerTy()};
1656     auto *FnTy =
1657         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
1658     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
1659     break;
1660   }
1661   case OMPRTL__kmpc_global_thread_num: {
1662     // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
1663     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1664     auto *FnTy =
1665         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1666     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
1667     break;
1668   }
1669   case OMPRTL__kmpc_threadprivate_cached: {
1670     // Build void *__kmpc_threadprivate_cached(ident_t *loc,
1671     // kmp_int32 global_tid, void *data, size_t size, void ***cache);
1672     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1673                                 CGM.VoidPtrTy, CGM.SizeTy,
1674                                 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1675     auto *FnTy =
1676         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
1677     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
1678     break;
1679   }
1680   case OMPRTL__kmpc_critical: {
1681     // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1682     // kmp_critical_name *crit);
1683     llvm::Type *TypeParams[] = {
1684         getIdentTyPointerTy(), CGM.Int32Ty,
1685         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1686     auto *FnTy =
1687         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1688     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
1689     break;
1690   }
1691   case OMPRTL__kmpc_critical_with_hint: {
1692     // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1693     // kmp_critical_name *crit, uintptr_t hint);
1694     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1695                                 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1696                                 CGM.IntPtrTy};
1697     auto *FnTy =
1698         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1699     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint");
1700     break;
1701   }
1702   case OMPRTL__kmpc_threadprivate_register: {
1703     // Build void __kmpc_threadprivate_register(ident_t *, void *data,
1704     // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
1705     // typedef void *(*kmpc_ctor)(void *);
1706     auto *KmpcCtorTy =
1707         llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1708                                 /*isVarArg*/ false)->getPointerTo();
1709     // typedef void *(*kmpc_cctor)(void *, void *);
1710     llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1711     auto *KmpcCopyCtorTy =
1712         llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1713                                 /*isVarArg*/ false)
1714             ->getPointerTo();
1715     // typedef void (*kmpc_dtor)(void *);
1716     auto *KmpcDtorTy =
1717         llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
1718             ->getPointerTo();
1719     llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
1720                               KmpcCopyCtorTy, KmpcDtorTy};
1721     auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1722                                         /*isVarArg*/ false);
1723     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
1724     break;
1725   }
1726   case OMPRTL__kmpc_end_critical: {
1727     // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1728     // kmp_critical_name *crit);
1729     llvm::Type *TypeParams[] = {
1730         getIdentTyPointerTy(), CGM.Int32Ty,
1731         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1732     auto *FnTy =
1733         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1734     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
1735     break;
1736   }
1737   case OMPRTL__kmpc_cancel_barrier: {
1738     // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
1739     // global_tid);
1740     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1741     auto *FnTy =
1742         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
1743     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
1744     break;
1745   }
1746   case OMPRTL__kmpc_barrier: {
1747     // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
1748     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1749     auto *FnTy =
1750         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1751     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
1752     break;
1753   }
1754   case OMPRTL__kmpc_for_static_fini: {
1755     // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
1756     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1757     auto *FnTy =
1758         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1759     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
1760     break;
1761   }
1762   case OMPRTL__kmpc_push_num_threads: {
1763     // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
1764     // kmp_int32 num_threads)
1765     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1766                                 CGM.Int32Ty};
1767     auto *FnTy =
1768         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1769     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
1770     break;
1771   }
1772   case OMPRTL__kmpc_serialized_parallel: {
1773     // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
1774     // global_tid);
1775     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1776     auto *FnTy =
1777         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1778     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
1779     break;
1780   }
1781   case OMPRTL__kmpc_end_serialized_parallel: {
1782     // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
1783     // global_tid);
1784     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1785     auto *FnTy =
1786         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1787     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
1788     break;
1789   }
1790   case OMPRTL__kmpc_flush: {
1791     // Build void __kmpc_flush(ident_t *loc);
1792     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
1793     auto *FnTy =
1794         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
1795     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
1796     break;
1797   }
1798   case OMPRTL__kmpc_master: {
1799     // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
1800     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1801     auto *FnTy =
1802         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1803     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
1804     break;
1805   }
1806   case OMPRTL__kmpc_end_master: {
1807     // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
1808     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1809     auto *FnTy =
1810         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1811     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
1812     break;
1813   }
1814   case OMPRTL__kmpc_omp_taskyield: {
1815     // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
1816     // int end_part);
1817     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
1818     auto *FnTy =
1819         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1820     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
1821     break;
1822   }
1823   case OMPRTL__kmpc_single: {
1824     // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
1825     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1826     auto *FnTy =
1827         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1828     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
1829     break;
1830   }
1831   case OMPRTL__kmpc_end_single: {
1832     // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
1833     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1834     auto *FnTy =
1835         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1836     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
1837     break;
1838   }
1839   case OMPRTL__kmpc_omp_task_alloc: {
1840     // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
1841     // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1842     // kmp_routine_entry_t *task_entry);
1843     assert(KmpRoutineEntryPtrTy != nullptr &&
1844            "Type kmp_routine_entry_t must be created.");
1845     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
1846                                 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1847     // Return void * and then cast to particular kmp_task_t type.
1848     auto *FnTy =
1849         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
1850     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
1851     break;
1852   }
1853   case OMPRTL__kmpc_omp_task: {
1854     // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1855     // *new_task);
1856     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1857                                 CGM.VoidPtrTy};
1858     auto *FnTy =
1859         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1860     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
1861     break;
1862   }
1863   case OMPRTL__kmpc_copyprivate: {
1864     // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
1865     // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
1866     // kmp_int32 didit);
1867     llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1868     auto *CpyFnTy =
1869         llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
1870     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
1871                                 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1872                                 CGM.Int32Ty};
1873     auto *FnTy =
1874         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1875     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
1876     break;
1877   }
1878   case OMPRTL__kmpc_reduce: {
1879     // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
1880     // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
1881     // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
1882     llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1883     auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1884                                                /*isVarArg=*/false);
1885     llvm::Type *TypeParams[] = {
1886         getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1887         CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1888         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1889     auto *FnTy =
1890         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1891     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
1892     break;
1893   }
1894   case OMPRTL__kmpc_reduce_nowait: {
1895     // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
1896     // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
1897     // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
1898     // *lck);
1899     llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1900     auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1901                                                /*isVarArg=*/false);
1902     llvm::Type *TypeParams[] = {
1903         getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
1904         CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1905         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1906     auto *FnTy =
1907         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1908     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
1909     break;
1910   }
1911   case OMPRTL__kmpc_end_reduce: {
1912     // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
1913     // kmp_critical_name *lck);
1914     llvm::Type *TypeParams[] = {
1915         getIdentTyPointerTy(), CGM.Int32Ty,
1916         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1917     auto *FnTy =
1918         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1919     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
1920     break;
1921   }
1922   case OMPRTL__kmpc_end_reduce_nowait: {
1923     // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
1924     // kmp_critical_name *lck);
1925     llvm::Type *TypeParams[] = {
1926         getIdentTyPointerTy(), CGM.Int32Ty,
1927         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1928     auto *FnTy =
1929         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1930     RTLFn =
1931         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
1932     break;
1933   }
1934   case OMPRTL__kmpc_omp_task_begin_if0: {
1935     // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1936     // *new_task);
1937     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1938                                 CGM.VoidPtrTy};
1939     auto *FnTy =
1940         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1941     RTLFn =
1942         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
1943     break;
1944   }
1945   case OMPRTL__kmpc_omp_task_complete_if0: {
1946     // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
1947     // *new_task);
1948     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
1949                                 CGM.VoidPtrTy};
1950     auto *FnTy =
1951         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1952     RTLFn = CGM.CreateRuntimeFunction(FnTy,
1953                                       /*Name=*/"__kmpc_omp_task_complete_if0");
1954     break;
1955   }
1956   case OMPRTL__kmpc_ordered: {
1957     // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
1958     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1959     auto *FnTy =
1960         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1961     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
1962     break;
1963   }
1964   case OMPRTL__kmpc_end_ordered: {
1965     // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
1966     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1967     auto *FnTy =
1968         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1969     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
1970     break;
1971   }
1972   case OMPRTL__kmpc_omp_taskwait: {
1973     // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
1974     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1975     auto *FnTy =
1976         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
1977     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
1978     break;
1979   }
1980   case OMPRTL__kmpc_taskgroup: {
1981     // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
1982     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1983     auto *FnTy =
1984         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1985     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
1986     break;
1987   }
1988   case OMPRTL__kmpc_end_taskgroup: {
1989     // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
1990     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
1991     auto *FnTy =
1992         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
1993     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
1994     break;
1995   }
1996   case OMPRTL__kmpc_push_proc_bind: {
1997     // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
1998     // int proc_bind)
1999     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2000     auto *FnTy =
2001         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2002     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
2003     break;
2004   }
2005   case OMPRTL__kmpc_omp_task_with_deps: {
2006     // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2007     // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2008     // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
2009     llvm::Type *TypeParams[] = {
2010         getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
2011         CGM.VoidPtrTy,         CGM.Int32Ty, CGM.VoidPtrTy};
2012     auto *FnTy =
2013         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
2014     RTLFn =
2015         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
2016     break;
2017   }
2018   case OMPRTL__kmpc_omp_wait_deps: {
2019     // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2020     // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
2021     // kmp_depend_info_t *noalias_dep_list);
2022     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2023                                 CGM.Int32Ty,           CGM.VoidPtrTy,
2024                                 CGM.Int32Ty,           CGM.VoidPtrTy};
2025     auto *FnTy =
2026         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2027     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
2028     break;
2029   }
2030   case OMPRTL__kmpc_cancellationpoint: {
2031     // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2032     // global_tid, kmp_int32 cncl_kind)
2033     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2034     auto *FnTy =
2035         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2036     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
2037     break;
2038   }
2039   case OMPRTL__kmpc_cancel: {
2040     // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2041     // kmp_int32 cncl_kind)
2042     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
2043     auto *FnTy =
2044         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2045     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
2046     break;
2047   }
2048   case OMPRTL__kmpc_push_num_teams: {
2049     // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid,
2050     // kmp_int32 num_teams, kmp_int32 num_threads)
2051     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
2052         CGM.Int32Ty};
2053     auto *FnTy =
2054         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2055     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams");
2056     break;
2057   }
2058   case OMPRTL__kmpc_fork_teams: {
2059     // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro
2060     // microtask, ...);
2061     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2062                                 getKmpc_MicroPointerTy()};
2063     auto *FnTy =
2064         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
2065     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams");
2066     break;
2067   }
2068   case OMPRTL__kmpc_taskloop: {
2069     // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
2070     // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
2071     // sched, kmp_uint64 grainsize, void *task_dup);
2072     llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2073                                 CGM.IntTy,
2074                                 CGM.VoidPtrTy,
2075                                 CGM.IntTy,
2076                                 CGM.Int64Ty->getPointerTo(),
2077                                 CGM.Int64Ty->getPointerTo(),
2078                                 CGM.Int64Ty,
2079                                 CGM.IntTy,
2080                                 CGM.IntTy,
2081                                 CGM.Int64Ty,
2082                                 CGM.VoidPtrTy};
2083     auto *FnTy =
2084         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2085     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop");
2086     break;
2087   }
2088   case OMPRTL__kmpc_doacross_init: {
2089     // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32
2090     // num_dims, struct kmp_dim *dims);
2091     llvm::Type *TypeParams[] = {getIdentTyPointerTy(),
2092                                 CGM.Int32Ty,
2093                                 CGM.Int32Ty,
2094                                 CGM.VoidPtrTy};
2095     auto *FnTy =
2096         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2097     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init");
2098     break;
2099   }
2100   case OMPRTL__kmpc_doacross_fini: {
2101     // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);
2102     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
2103     auto *FnTy =
2104         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2105     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini");
2106     break;
2107   }
2108   case OMPRTL__kmpc_doacross_post: {
2109     // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64
2110     // *vec);
2111     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2112                                 CGM.Int64Ty->getPointerTo()};
2113     auto *FnTy =
2114         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2115     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post");
2116     break;
2117   }
2118   case OMPRTL__kmpc_doacross_wait: {
2119     // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64
2120     // *vec);
2121     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
2122                                 CGM.Int64Ty->getPointerTo()};
2123     auto *FnTy =
2124         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2125     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait");
2126     break;
2127   }
2128   case OMPRTL__kmpc_task_reduction_init: {
2129     // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void
2130     // *data);
2131     llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy};
2132     auto *FnTy =
2133         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2134     RTLFn =
2135         CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init");
2136     break;
2137   }
2138   case OMPRTL__kmpc_task_reduction_get_th_data: {
2139     // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
2140     // *d);
2141     llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy};
2142     auto *FnTy =
2143         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
2144     RTLFn = CGM.CreateRuntimeFunction(
2145         FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data");
2146     break;
2147   }
2148   case OMPRTL__tgt_target: {
2149     // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t
2150     // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2151     // *arg_types);
2152     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2153                                 CGM.VoidPtrTy,
2154                                 CGM.Int32Ty,
2155                                 CGM.VoidPtrPtrTy,
2156                                 CGM.VoidPtrPtrTy,
2157                                 CGM.SizeTy->getPointerTo(),
2158                                 CGM.Int64Ty->getPointerTo()};
2159     auto *FnTy =
2160         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2161     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target");
2162     break;
2163   }
2164   case OMPRTL__tgt_target_nowait: {
2165     // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr,
2166     // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2167     // int64_t *arg_types);
2168     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2169                                 CGM.VoidPtrTy,
2170                                 CGM.Int32Ty,
2171                                 CGM.VoidPtrPtrTy,
2172                                 CGM.VoidPtrPtrTy,
2173                                 CGM.SizeTy->getPointerTo(),
2174                                 CGM.Int64Ty->getPointerTo()};
2175     auto *FnTy =
2176         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2177     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait");
2178     break;
2179   }
2180   case OMPRTL__tgt_target_teams: {
2181     // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr,
2182     // int32_t arg_num, void** args_base, void **args, size_t *arg_sizes,
2183     // int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2184     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2185                                 CGM.VoidPtrTy,
2186                                 CGM.Int32Ty,
2187                                 CGM.VoidPtrPtrTy,
2188                                 CGM.VoidPtrPtrTy,
2189                                 CGM.SizeTy->getPointerTo(),
2190                                 CGM.Int64Ty->getPointerTo(),
2191                                 CGM.Int32Ty,
2192                                 CGM.Int32Ty};
2193     auto *FnTy =
2194         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2195     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams");
2196     break;
2197   }
2198   case OMPRTL__tgt_target_teams_nowait: {
2199     // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void
2200     // *host_ptr, int32_t arg_num, void** args_base, void **args, size_t
2201     // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit);
2202     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2203                                 CGM.VoidPtrTy,
2204                                 CGM.Int32Ty,
2205                                 CGM.VoidPtrPtrTy,
2206                                 CGM.VoidPtrPtrTy,
2207                                 CGM.SizeTy->getPointerTo(),
2208                                 CGM.Int64Ty->getPointerTo(),
2209                                 CGM.Int32Ty,
2210                                 CGM.Int32Ty};
2211     auto *FnTy =
2212         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2213     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait");
2214     break;
2215   }
2216   case OMPRTL__tgt_register_lib: {
2217     // Build void __tgt_register_lib(__tgt_bin_desc *desc);
2218     QualType ParamTy =
2219         CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2220     llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2221     auto *FnTy =
2222         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2223     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib");
2224     break;
2225   }
2226   case OMPRTL__tgt_unregister_lib: {
2227     // Build void __tgt_unregister_lib(__tgt_bin_desc *desc);
2228     QualType ParamTy =
2229         CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy());
2230     llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
2231     auto *FnTy =
2232         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2233     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib");
2234     break;
2235   }
2236   case OMPRTL__tgt_target_data_begin: {
2237     // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
2238     // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2239     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2240                                 CGM.Int32Ty,
2241                                 CGM.VoidPtrPtrTy,
2242                                 CGM.VoidPtrPtrTy,
2243                                 CGM.SizeTy->getPointerTo(),
2244                                 CGM.Int64Ty->getPointerTo()};
2245     auto *FnTy =
2246         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2247     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin");
2248     break;
2249   }
2250   case OMPRTL__tgt_target_data_begin_nowait: {
2251     // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t
2252     // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2253     // *arg_types);
2254     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2255                                 CGM.Int32Ty,
2256                                 CGM.VoidPtrPtrTy,
2257                                 CGM.VoidPtrPtrTy,
2258                                 CGM.SizeTy->getPointerTo(),
2259                                 CGM.Int64Ty->getPointerTo()};
2260     auto *FnTy =
2261         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2262     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait");
2263     break;
2264   }
2265   case OMPRTL__tgt_target_data_end: {
2266     // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
2267     // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2268     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2269                                 CGM.Int32Ty,
2270                                 CGM.VoidPtrPtrTy,
2271                                 CGM.VoidPtrPtrTy,
2272                                 CGM.SizeTy->getPointerTo(),
2273                                 CGM.Int64Ty->getPointerTo()};
2274     auto *FnTy =
2275         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2276     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end");
2277     break;
2278   }
2279   case OMPRTL__tgt_target_data_end_nowait: {
2280     // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t
2281     // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2282     // *arg_types);
2283     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2284                                 CGM.Int32Ty,
2285                                 CGM.VoidPtrPtrTy,
2286                                 CGM.VoidPtrPtrTy,
2287                                 CGM.SizeTy->getPointerTo(),
2288                                 CGM.Int64Ty->getPointerTo()};
2289     auto *FnTy =
2290         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2291     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait");
2292     break;
2293   }
2294   case OMPRTL__tgt_target_data_update: {
2295     // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
2296     // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types);
2297     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2298                                 CGM.Int32Ty,
2299                                 CGM.VoidPtrPtrTy,
2300                                 CGM.VoidPtrPtrTy,
2301                                 CGM.SizeTy->getPointerTo(),
2302                                 CGM.Int64Ty->getPointerTo()};
2303     auto *FnTy =
2304         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2305     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update");
2306     break;
2307   }
2308   case OMPRTL__tgt_target_data_update_nowait: {
2309     // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t
2310     // arg_num, void** args_base, void **args, size_t *arg_sizes, int64_t
2311     // *arg_types);
2312     llvm::Type *TypeParams[] = {CGM.Int64Ty,
2313                                 CGM.Int32Ty,
2314                                 CGM.VoidPtrPtrTy,
2315                                 CGM.VoidPtrPtrTy,
2316                                 CGM.SizeTy->getPointerTo(),
2317                                 CGM.Int64Ty->getPointerTo()};
2318     auto *FnTy =
2319         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2320     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait");
2321     break;
2322   }
2323   }
2324   assert(RTLFn && "Unable to find OpenMP runtime function");
2325   return RTLFn;
2326 }
2327
2328 llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize,
2329                                                              bool IVSigned) {
2330   assert((IVSize == 32 || IVSize == 64) &&
2331          "IV size is not compatible with the omp runtime");
2332   StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
2333                                             : "__kmpc_for_static_init_4u")
2334                                 : (IVSigned ? "__kmpc_for_static_init_8"
2335                                             : "__kmpc_for_static_init_8u");
2336   llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2337   auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2338   llvm::Type *TypeParams[] = {
2339     getIdentTyPointerTy(),                     // loc
2340     CGM.Int32Ty,                               // tid
2341     CGM.Int32Ty,                               // schedtype
2342     llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2343     PtrTy,                                     // p_lower
2344     PtrTy,                                     // p_upper
2345     PtrTy,                                     // p_stride
2346     ITy,                                       // incr
2347     ITy                                        // chunk
2348   };
2349   auto *FnTy =
2350       llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2351   return CGM.CreateRuntimeFunction(FnTy, Name);
2352 }
2353
2354 llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize,
2355                                                             bool IVSigned) {
2356   assert((IVSize == 32 || IVSize == 64) &&
2357          "IV size is not compatible with the omp runtime");
2358   StringRef Name =
2359       IVSize == 32
2360           ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
2361           : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
2362   llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2363   llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
2364                                CGM.Int32Ty,           // tid
2365                                CGM.Int32Ty,           // schedtype
2366                                ITy,                   // lower
2367                                ITy,                   // upper
2368                                ITy,                   // stride
2369                                ITy                    // chunk
2370   };
2371   auto *FnTy =
2372       llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
2373   return CGM.CreateRuntimeFunction(FnTy, Name);
2374 }
2375
2376 llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize,
2377                                                             bool IVSigned) {
2378   assert((IVSize == 32 || IVSize == 64) &&
2379          "IV size is not compatible with the omp runtime");
2380   StringRef Name =
2381       IVSize == 32
2382           ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
2383           : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
2384   llvm::Type *TypeParams[] = {
2385       getIdentTyPointerTy(), // loc
2386       CGM.Int32Ty,           // tid
2387   };
2388   auto *FnTy =
2389       llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
2390   return CGM.CreateRuntimeFunction(FnTy, Name);
2391 }
2392
2393 llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize,
2394                                                             bool IVSigned) {
2395   assert((IVSize == 32 || IVSize == 64) &&
2396          "IV size is not compatible with the omp runtime");
2397   StringRef Name =
2398       IVSize == 32
2399           ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
2400           : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
2401   llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
2402   auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2403   llvm::Type *TypeParams[] = {
2404     getIdentTyPointerTy(),                     // loc
2405     CGM.Int32Ty,                               // tid
2406     llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
2407     PtrTy,                                     // p_lower
2408     PtrTy,                                     // p_upper
2409     PtrTy                                      // p_stride
2410   };
2411   auto *FnTy =
2412       llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
2413   return CGM.CreateRuntimeFunction(FnTy, Name);
2414 }
2415
2416 Address CGOpenMPRuntime::getAddrOfDeclareTargetLink(const VarDecl *VD) {
2417   if (CGM.getLangOpts().OpenMPSimd)
2418     return Address::invalid();
2419   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2420       isDeclareTargetDeclaration(VD);
2421   if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Link) {
2422     SmallString<64> PtrName;
2423     {
2424       llvm::raw_svector_ostream OS(PtrName);
2425       OS << CGM.getMangledName(GlobalDecl(VD)) << "_decl_tgt_link_ptr";
2426     }
2427     llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName);
2428     if (!Ptr) {
2429       QualType PtrTy = CGM.getContext().getPointerType(VD->getType());
2430       Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy),
2431                                         PtrName);
2432       if (!CGM.getLangOpts().OpenMPIsDevice) {
2433         auto *GV = cast<llvm::GlobalVariable>(Ptr);
2434         GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2435         GV->setInitializer(CGM.GetAddrOfGlobal(VD));
2436       }
2437       CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ptr));
2438       registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr));
2439     }
2440     return Address(Ptr, CGM.getContext().getDeclAlign(VD));
2441   }
2442   return Address::invalid();
2443 }
2444
2445 llvm::Constant *
2446 CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
2447   assert(!CGM.getLangOpts().OpenMPUseTLS ||
2448          !CGM.getContext().getTargetInfo().isTLSSupported());
2449   // Lookup the entry, lazily creating it if necessary.
2450   std::string Suffix = getName({"cache", ""});
2451   return getOrCreateInternalVariable(
2452       CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix));
2453 }
2454
2455 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
2456                                                 const VarDecl *VD,
2457                                                 Address VDAddr,
2458                                                 SourceLocation Loc) {
2459   if (CGM.getLangOpts().OpenMPUseTLS &&
2460       CGM.getContext().getTargetInfo().isTLSSupported())
2461     return VDAddr;
2462
2463   llvm::Type *VarTy = VDAddr.getElementType();
2464   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2465                          CGF.Builder.CreatePointerCast(VDAddr.getPointer(),
2466                                                        CGM.Int8PtrTy),
2467                          CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
2468                          getOrCreateThreadPrivateCache(VD)};
2469   return Address(CGF.EmitRuntimeCall(
2470       createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2471                  VDAddr.getAlignment());
2472 }
2473
2474 void CGOpenMPRuntime::emitThreadPrivateVarInit(
2475     CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
2476     llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
2477   // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
2478   // library.
2479   llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
2480   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
2481                       OMPLoc);
2482   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
2483   // to register constructor/destructor for variable.
2484   llvm::Value *Args[] = {
2485       OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
2486       Ctor, CopyCtor, Dtor};
2487   CGF.EmitRuntimeCall(
2488       createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
2489 }
2490
2491 llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
2492     const VarDecl *VD, Address VDAddr, SourceLocation Loc,
2493     bool PerformInit, CodeGenFunction *CGF) {
2494   if (CGM.getLangOpts().OpenMPUseTLS &&
2495       CGM.getContext().getTargetInfo().isTLSSupported())
2496     return nullptr;
2497
2498   VD = VD->getDefinition(CGM.getContext());
2499   if (VD && ThreadPrivateWithDefinition.count(VD) == 0) {
2500     ThreadPrivateWithDefinition.insert(VD);
2501     QualType ASTTy = VD->getType();
2502
2503     llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
2504     const Expr *Init = VD->getAnyInitializer();
2505     if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2506       // Generate function that re-emits the declaration's initializer into the
2507       // threadprivate copy of the variable VD
2508       CodeGenFunction CtorCGF(CGM);
2509       FunctionArgList Args;
2510       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2511                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2512                             ImplicitParamDecl::Other);
2513       Args.push_back(&Dst);
2514
2515       const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2516           CGM.getContext().VoidPtrTy, Args);
2517       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2518       std::string Name = getName({"__kmpc_global_ctor_", ""});
2519       llvm::Function *Fn =
2520           CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2521       CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
2522                             Args, Loc, Loc);
2523       llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
2524           CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2525           CGM.getContext().VoidPtrTy, Dst.getLocation());
2526       Address Arg = Address(ArgVal, VDAddr.getAlignment());
2527       Arg = CtorCGF.Builder.CreateElementBitCast(
2528           Arg, CtorCGF.ConvertTypeForMem(ASTTy));
2529       CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
2530                                /*IsInitializer=*/true);
2531       ArgVal = CtorCGF.EmitLoadOfScalar(
2532           CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false,
2533           CGM.getContext().VoidPtrTy, Dst.getLocation());
2534       CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
2535       CtorCGF.FinishFunction();
2536       Ctor = Fn;
2537     }
2538     if (VD->getType().isDestructedType() != QualType::DK_none) {
2539       // Generate function that emits destructor call for the threadprivate copy
2540       // of the variable VD
2541       CodeGenFunction DtorCGF(CGM);
2542       FunctionArgList Args;
2543       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc,
2544                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy,
2545                             ImplicitParamDecl::Other);
2546       Args.push_back(&Dst);
2547
2548       const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
2549           CGM.getContext().VoidTy, Args);
2550       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2551       std::string Name = getName({"__kmpc_global_dtor_", ""});
2552       llvm::Function *Fn =
2553           CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc);
2554       auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2555       DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
2556                             Loc, Loc);
2557       // Create a scope with an artificial location for the body of this function.
2558       auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2559       llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
2560           DtorCGF.GetAddrOfLocalVar(&Dst),
2561           /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation());
2562       DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy,
2563                           DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2564                           DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2565       DtorCGF.FinishFunction();
2566       Dtor = Fn;
2567     }
2568     // Do not emit init function if it is not required.
2569     if (!Ctor && !Dtor)
2570       return nullptr;
2571
2572     llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
2573     auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
2574                                                /*isVarArg=*/false)
2575                            ->getPointerTo();
2576     // Copying constructor for the threadprivate variable.
2577     // Must be NULL - reserved by runtime, but currently it requires that this
2578     // parameter is always NULL. Otherwise it fires assertion.
2579     CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2580     if (Ctor == nullptr) {
2581       auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
2582                                              /*isVarArg=*/false)
2583                          ->getPointerTo();
2584       Ctor = llvm::Constant::getNullValue(CtorTy);
2585     }
2586     if (Dtor == nullptr) {
2587       auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
2588                                              /*isVarArg=*/false)
2589                          ->getPointerTo();
2590       Dtor = llvm::Constant::getNullValue(DtorTy);
2591     }
2592     if (!CGF) {
2593       auto *InitFunctionTy =
2594           llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
2595       std::string Name = getName({"__omp_threadprivate_init_", ""});
2596       llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction(
2597           InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction());
2598       CodeGenFunction InitCGF(CGM);
2599       FunctionArgList ArgList;
2600       InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
2601                             CGM.getTypes().arrangeNullaryFunction(), ArgList,
2602                             Loc, Loc);
2603       emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2604       InitCGF.FinishFunction();
2605       return InitFunction;
2606     }
2607     emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
2608   }
2609   return nullptr;
2610 }
2611
2612 /// Obtain information that uniquely identifies a target entry. This
2613 /// consists of the file and device IDs as well as line number associated with
2614 /// the relevant entry source location.
2615 static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc,
2616                                      unsigned &DeviceID, unsigned &FileID,
2617                                      unsigned &LineNum) {
2618   SourceManager &SM = C.getSourceManager();
2619
2620   // The loc should be always valid and have a file ID (the user cannot use
2621   // #pragma directives in macros)
2622
2623   assert(Loc.isValid() && "Source location is expected to be always valid.");
2624
2625   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
2626   assert(PLoc.isValid() && "Source location is expected to be always valid.");
2627
2628   llvm::sys::fs::UniqueID ID;
2629   if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
2630     SM.getDiagnostics().Report(diag::err_cannot_open_file)
2631         << PLoc.getFilename() << EC.message();
2632
2633   DeviceID = ID.getDevice();
2634   FileID = ID.getFile();
2635   LineNum = PLoc.getLine();
2636 }
2637
2638 bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
2639                                                      llvm::GlobalVariable *Addr,
2640                                                      bool PerformInit) {
2641   Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2642       isDeclareTargetDeclaration(VD);
2643   if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link)
2644     return false;
2645   VD = VD->getDefinition(CGM.getContext());
2646   if (VD && !DeclareTargetWithDefinition.insert(VD).second)
2647     return CGM.getLangOpts().OpenMPIsDevice;
2648
2649   QualType ASTTy = VD->getType();
2650
2651   SourceLocation Loc = VD->getCanonicalDecl()->getLocStart();
2652   // Produce the unique prefix to identify the new target regions. We use
2653   // the source location of the variable declaration which we know to not
2654   // conflict with any target region.
2655   unsigned DeviceID;
2656   unsigned FileID;
2657   unsigned Line;
2658   getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line);
2659   SmallString<128> Buffer, Out;
2660   {
2661     llvm::raw_svector_ostream OS(Buffer);
2662     OS << "__omp_offloading_" << llvm::format("_%x", DeviceID)
2663        << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line;
2664   }
2665
2666   const Expr *Init = VD->getAnyInitializer();
2667   if (CGM.getLangOpts().CPlusPlus && PerformInit) {
2668     llvm::Constant *Ctor;
2669     llvm::Constant *ID;
2670     if (CGM.getLangOpts().OpenMPIsDevice) {
2671       // Generate function that re-emits the declaration's initializer into
2672       // the threadprivate copy of the variable VD
2673       CodeGenFunction CtorCGF(CGM);
2674
2675       const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2676       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2677       llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2678           FTy, Twine(Buffer, "_ctor"), FI, Loc);
2679       auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF);
2680       CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2681                             FunctionArgList(), Loc, Loc);
2682       auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF);
2683       CtorCGF.EmitAnyExprToMem(Init,
2684                                Address(Addr, CGM.getContext().getDeclAlign(VD)),
2685                                Init->getType().getQualifiers(),
2686                                /*IsInitializer=*/true);
2687       CtorCGF.FinishFunction();
2688       Ctor = Fn;
2689       ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2690       CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor));
2691     } else {
2692       Ctor = new llvm::GlobalVariable(
2693           CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2694           llvm::GlobalValue::PrivateLinkage,
2695           llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor"));
2696       ID = Ctor;
2697     }
2698
2699     // Register the information for the entry associated with the constructor.
2700     Out.clear();
2701     OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2702         DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor,
2703         ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor);
2704   }
2705   if (VD->getType().isDestructedType() != QualType::DK_none) {
2706     llvm::Constant *Dtor;
2707     llvm::Constant *ID;
2708     if (CGM.getLangOpts().OpenMPIsDevice) {
2709       // Generate function that emits destructor call for the threadprivate
2710       // copy of the variable VD
2711       CodeGenFunction DtorCGF(CGM);
2712
2713       const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
2714       llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
2715       llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction(
2716           FTy, Twine(Buffer, "_dtor"), FI, Loc);
2717       auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF);
2718       DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI,
2719                             FunctionArgList(), Loc, Loc);
2720       // Create a scope with an artificial location for the body of this
2721       // function.
2722       auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
2723       DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)),
2724                           ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()),
2725                           DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
2726       DtorCGF.FinishFunction();
2727       Dtor = Fn;
2728       ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy);
2729       CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor));
2730     } else {
2731       Dtor = new llvm::GlobalVariable(
2732           CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
2733           llvm::GlobalValue::PrivateLinkage,
2734           llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor"));
2735       ID = Dtor;
2736     }
2737     // Register the information for the entry associated with the destructor.
2738     Out.clear();
2739     OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
2740         DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor,
2741         ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor);
2742   }
2743   return CGM.getLangOpts().OpenMPIsDevice;
2744 }
2745
2746 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2747                                                           QualType VarType,
2748                                                           StringRef Name) {
2749   std::string Suffix = getName({"artificial", ""});
2750   std::string CacheSuffix = getName({"cache", ""});
2751   llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType);
2752   llvm::Value *GAddr =
2753       getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix));
2754   llvm::Value *Args[] = {
2755       emitUpdateLocation(CGF, SourceLocation()),
2756       getThreadID(CGF, SourceLocation()),
2757       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy),
2758       CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy,
2759                                 /*IsSigned=*/false),
2760       getOrCreateInternalVariable(
2761           CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))};
2762   return Address(
2763       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2764           CGF.EmitRuntimeCall(
2765               createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args),
2766           VarLVType->getPointerTo(/*AddrSpace=*/0)),
2767       CGM.getPointerAlign());
2768 }
2769
2770 void CGOpenMPRuntime::emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
2771                                       const RegionCodeGenTy &ThenGen,
2772                                       const RegionCodeGenTy &ElseGen) {
2773   CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
2774
2775   // If the condition constant folds and can be elided, try to avoid emitting
2776   // the condition and the dead arm of the if/else.
2777   bool CondConstant;
2778   if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
2779     if (CondConstant)
2780       ThenGen(CGF);
2781     else
2782       ElseGen(CGF);
2783     return;
2784   }
2785
2786   // Otherwise, the condition did not fold, or we couldn't elide it.  Just
2787   // emit the conditional branch.
2788   llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then");
2789   llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else");
2790   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end");
2791   CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
2792
2793   // Emit the 'then' code.
2794   CGF.EmitBlock(ThenBlock);
2795   ThenGen(CGF);
2796   CGF.EmitBranch(ContBlock);
2797   // Emit the 'else' code if present.
2798   // There is no need to emit line number for unconditional branch.
2799   (void)ApplyDebugLocation::CreateEmpty(CGF);
2800   CGF.EmitBlock(ElseBlock);
2801   ElseGen(CGF);
2802   // There is no need to emit line number for unconditional branch.
2803   (void)ApplyDebugLocation::CreateEmpty(CGF);
2804   CGF.EmitBranch(ContBlock);
2805   // Emit the continuation block for code after the if.
2806   CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
2807 }
2808
2809 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
2810                                        llvm::Value *OutlinedFn,
2811                                        ArrayRef<llvm::Value *> CapturedVars,
2812                                        const Expr *IfCond) {
2813   if (!CGF.HaveInsertPoint())
2814     return;
2815   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2816   auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF,
2817                                                      PrePostActionTy &) {
2818     // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2819     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2820     llvm::Value *Args[] = {
2821         RTLoc,
2822         CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
2823         CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
2824     llvm::SmallVector<llvm::Value *, 16> RealArgs;
2825     RealArgs.append(std::begin(Args), std::end(Args));
2826     RealArgs.append(CapturedVars.begin(), CapturedVars.end());
2827
2828     llvm::Value *RTLFn = RT.createRuntimeFunction(OMPRTL__kmpc_fork_call);
2829     CGF.EmitRuntimeCall(RTLFn, RealArgs);
2830   };
2831   auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF,
2832                                                           PrePostActionTy &) {
2833     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2834     llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2835     // Build calls:
2836     // __kmpc_serialized_parallel(&Loc, GTid);
2837     llvm::Value *Args[] = {RTLoc, ThreadID};
2838     CGF.EmitRuntimeCall(
2839         RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args);
2840
2841     // OutlinedFn(&GTid, &zero, CapturedStruct);
2842     Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2843                                                         /*Name*/ ".zero.addr");
2844     CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
2845     llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2846     // ThreadId for serialized parallels is 0.
2847     OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2848     OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2849     OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2850     RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2851
2852     // __kmpc_end_serialized_parallel(&Loc, GTid);
2853     llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2854     CGF.EmitRuntimeCall(
2855         RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel),
2856         EndArgs);
2857   };
2858   if (IfCond) {
2859     emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
2860   } else {
2861     RegionCodeGenTy ThenRCG(ThenGen);
2862     ThenRCG(CGF);
2863   }
2864 }
2865
2866 // If we're inside an (outlined) parallel region, use the region info's
2867 // thread-ID variable (it is passed in a first argument of the outlined function
2868 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2869 // regular serial code region, get thread ID by calling kmp_int32
2870 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2871 // return the address of that temp.
2872 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2873                                              SourceLocation Loc) {
2874   if (auto *OMPRegionInfo =
2875           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
2876     if (OMPRegionInfo->getThreadIDVariable())
2877       return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2878
2879   llvm::Value *ThreadID = getThreadID(CGF, Loc);
2880   QualType Int32Ty =
2881       CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2882   Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
2883   CGF.EmitStoreOfScalar(ThreadID,
2884                         CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty));
2885
2886   return ThreadIDTemp;
2887 }
2888
2889 llvm::Constant *
2890 CGOpenMPRuntime::getOrCreateInternalVariable(llvm::Type *Ty,
2891                                              const llvm::Twine &Name) {
2892   SmallString<256> Buffer;
2893   llvm::raw_svector_ostream Out(Buffer);
2894   Out << Name;
2895   StringRef RuntimeName = Out.str();
2896   auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
2897   if (Elem.second) {
2898     assert(Elem.second->getType()->getPointerElementType() == Ty &&
2899            "OMP internal variable has different type than requested");
2900     return &*Elem.second;
2901   }
2902
2903   return Elem.second = new llvm::GlobalVariable(
2904              CGM.getModule(), Ty, /*IsConstant*/ false,
2905              llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2906              Elem.first());
2907 }
2908
2909 llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2910   std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2911   std::string Name = getName({Prefix, "var"});
2912   return getOrCreateInternalVariable(KmpCriticalNameTy, Name);
2913 }
2914
2915 namespace {
2916 /// Common pre(post)-action for different OpenMP constructs.
2917 class CommonActionTy final : public PrePostActionTy {
2918   llvm::Value *EnterCallee;
2919   ArrayRef<llvm::Value *> EnterArgs;
2920   llvm::Value *ExitCallee;
2921   ArrayRef<llvm::Value *> ExitArgs;
2922   bool Conditional;
2923   llvm::BasicBlock *ContBlock = nullptr;
2924
2925 public:
2926   CommonActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs,
2927                  llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs,
2928                  bool Conditional = false)
2929       : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2930         ExitArgs(ExitArgs), Conditional(Conditional) {}
2931   void Enter(CodeGenFunction &CGF) override {
2932     llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
2933     if (Conditional) {
2934       llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
2935       auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
2936       ContBlock = CGF.createBasicBlock("omp_if.end");
2937       // Generate the branch (If-stmt)
2938       CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2939       CGF.EmitBlock(ThenBlock);
2940     }
2941   }
2942   void Done(CodeGenFunction &CGF) {
2943     // Emit the rest of blocks/branches
2944     CGF.EmitBranch(ContBlock);
2945     CGF.EmitBlock(ContBlock, true);
2946   }
2947   void Exit(CodeGenFunction &CGF) override {
2948     CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
2949   }
2950 };
2951 } // anonymous namespace
2952
2953 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2954                                          StringRef CriticalName,
2955                                          const RegionCodeGenTy &CriticalOpGen,
2956                                          SourceLocation Loc, const Expr *Hint) {
2957   // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2958   // CriticalOpGen();
2959   // __kmpc_end_critical(ident_t *, gtid, Lock);
2960   // Prepare arguments and build a call to __kmpc_critical
2961   if (!CGF.HaveInsertPoint())
2962     return;
2963   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2964                          getCriticalRegionLock(CriticalName)};
2965   llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args),
2966                                                 std::end(Args));
2967   if (Hint) {
2968     EnterArgs.push_back(CGF.Builder.CreateIntCast(
2969         CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false));
2970   }
2971   CommonActionTy Action(
2972       createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint
2973                                  : OMPRTL__kmpc_critical),
2974       EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args);
2975   CriticalOpGen.setAction(Action);
2976   emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
2977 }
2978
2979 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2980                                        const RegionCodeGenTy &MasterOpGen,
2981                                        SourceLocation Loc) {
2982   if (!CGF.HaveInsertPoint())
2983     return;
2984   // if(__kmpc_master(ident_t *, gtid)) {
2985   //   MasterOpGen();
2986   //   __kmpc_end_master(ident_t *, gtid);
2987   // }
2988   // Prepare arguments and build a call to __kmpc_master
2989   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2990   CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args,
2991                         createRuntimeFunction(OMPRTL__kmpc_end_master), Args,
2992                         /*Conditional=*/true);
2993   MasterOpGen.setAction(Action);
2994   emitInlinedDirective(CGF, OMPD_master, MasterOpGen);
2995   Action.Done(CGF);
2996 }
2997
2998 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2999                                         SourceLocation Loc) {
3000   if (!CGF.HaveInsertPoint())
3001     return;
3002   // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
3003   llvm::Value *Args[] = {
3004       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3005       llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
3006   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
3007   if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
3008     Region->emitUntiedSwitch(CGF);
3009 }
3010
3011 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
3012                                           const RegionCodeGenTy &TaskgroupOpGen,
3013                                           SourceLocation Loc) {
3014   if (!CGF.HaveInsertPoint())
3015     return;
3016   // __kmpc_taskgroup(ident_t *, gtid);
3017   // TaskgroupOpGen();
3018   // __kmpc_end_taskgroup(ident_t *, gtid);
3019   // Prepare arguments and build a call to __kmpc_taskgroup
3020   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3021   CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args,
3022                         createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
3023                         Args);
3024   TaskgroupOpGen.setAction(Action);
3025   emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
3026 }
3027
3028 /// Given an array of pointers to variables, project the address of a
3029 /// given variable.
3030 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
3031                                       unsigned Index, const VarDecl *Var) {
3032   // Pull out the pointer to the variable.
3033   Address PtrAddr =
3034       CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize());
3035   llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr);
3036
3037   Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var));
3038   Addr = CGF.Builder.CreateElementBitCast(
3039       Addr, CGF.ConvertTypeForMem(Var->getType()));
3040   return Addr;
3041 }
3042
3043 static llvm::Value *emitCopyprivateCopyFunction(
3044     CodeGenModule &CGM, llvm::Type *ArgsType,
3045     ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3046     ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3047     SourceLocation Loc) {
3048   ASTContext &C = CGM.getContext();
3049   // void copy_func(void *LHSArg, void *RHSArg);
3050   FunctionArgList Args;
3051   ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3052                            ImplicitParamDecl::Other);
3053   ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
3054                            ImplicitParamDecl::Other);
3055   Args.push_back(&LHSArg);
3056   Args.push_back(&RHSArg);
3057   const auto &CGFI =
3058       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3059   std::string Name =
3060       CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"});
3061   auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
3062                                     llvm::GlobalValue::InternalLinkage, Name,
3063                                     &CGM.getModule());
3064   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
3065   Fn->setDoesNotRecurse();
3066   CodeGenFunction CGF(CGM);
3067   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
3068   // Dest = (void*[n])(LHSArg);
3069   // Src = (void*[n])(RHSArg);
3070   Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3071       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
3072       ArgsType), CGF.getPointerAlign());
3073   Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3074       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
3075       ArgsType), CGF.getPointerAlign());
3076   // *(Type0*)Dst[0] = *(Type0*)Src[0];
3077   // *(Type1*)Dst[1] = *(Type1*)Src[1];
3078   // ...
3079   // *(Typen*)Dst[n] = *(Typen*)Src[n];
3080   for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3081     const auto *DestVar =
3082         cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3083     Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar);
3084
3085     const auto *SrcVar =
3086         cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3087     Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar);
3088
3089     const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3090     QualType Type = VD->getType();
3091     CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3092   }
3093   CGF.FinishFunction();
3094   return Fn;
3095 }
3096
3097 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
3098                                        const RegionCodeGenTy &SingleOpGen,
3099                                        SourceLocation Loc,
3100                                        ArrayRef<const Expr *> CopyprivateVars,
3101                                        ArrayRef<const Expr *> SrcExprs,
3102                                        ArrayRef<const Expr *> DstExprs,
3103                                        ArrayRef<const Expr *> AssignmentOps) {
3104   if (!CGF.HaveInsertPoint())
3105     return;
3106   assert(CopyprivateVars.size() == SrcExprs.size() &&
3107          CopyprivateVars.size() == DstExprs.size() &&
3108          CopyprivateVars.size() == AssignmentOps.size());
3109   ASTContext &C = CGM.getContext();
3110   // int32 did_it = 0;
3111   // if(__kmpc_single(ident_t *, gtid)) {
3112   //   SingleOpGen();
3113   //   __kmpc_end_single(ident_t *, gtid);
3114   //   did_it = 1;
3115   // }
3116   // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3117   // <copy_func>, did_it);
3118
3119   Address DidIt = Address::invalid();
3120   if (!CopyprivateVars.empty()) {
3121     // int32 did_it = 0;
3122     QualType KmpInt32Ty =
3123         C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3124     DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
3125     CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt);
3126   }
3127   // Prepare arguments and build a call to __kmpc_single
3128   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3129   CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args,
3130                         createRuntimeFunction(OMPRTL__kmpc_end_single), Args,
3131                         /*Conditional=*/true);
3132   SingleOpGen.setAction(Action);
3133   emitInlinedDirective(CGF, OMPD_single, SingleOpGen);
3134   if (DidIt.isValid()) {
3135     // did_it = 1;
3136     CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt);
3137   }
3138   Action.Done(CGF);
3139   // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
3140   // <copy_func>, did_it);
3141   if (DidIt.isValid()) {
3142     llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
3143     QualType CopyprivateArrayTy =
3144         C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
3145                                /*IndexTypeQuals=*/0);
3146     // Create a list of all private variables for copyprivate.
3147     Address CopyprivateList =
3148         CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
3149     for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3150       Address Elem = CGF.Builder.CreateConstArrayGEP(
3151           CopyprivateList, I, CGF.getPointerSize());
3152       CGF.Builder.CreateStore(
3153           CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3154               CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy),
3155           Elem);
3156     }
3157     // Build function that copies private values from single region to all other
3158     // threads in the corresponding parallel region.
3159     llvm::Value *CpyFn = emitCopyprivateCopyFunction(
3160         CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
3161         CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3162     llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy);
3163     Address CL =
3164       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
3165                                                       CGF.VoidPtrTy);
3166     llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt);
3167     llvm::Value *Args[] = {
3168         emitUpdateLocation(CGF, Loc), // ident_t *<loc>
3169         getThreadID(CGF, Loc),        // i32 <gtid>
3170         BufSize,                      // size_t <buf_size>
3171         CL.getPointer(),              // void *<copyprivate list>
3172         CpyFn,                        // void (*) (void *, void *) <copy_func>
3173         DidItVal                      // i32 did_it
3174     };
3175     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
3176   }
3177 }
3178
3179 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
3180                                         const RegionCodeGenTy &OrderedOpGen,
3181                                         SourceLocation Loc, bool IsThreads) {
3182   if (!CGF.HaveInsertPoint())
3183     return;
3184   // __kmpc_ordered(ident_t *, gtid);
3185   // OrderedOpGen();
3186   // __kmpc_end_ordered(ident_t *, gtid);
3187   // Prepare arguments and build a call to __kmpc_ordered
3188   if (IsThreads) {
3189     llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3190     CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args,
3191                           createRuntimeFunction(OMPRTL__kmpc_end_ordered),
3192                           Args);
3193     OrderedOpGen.setAction(Action);
3194     emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3195     return;
3196   }
3197   emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
3198 }
3199
3200 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
3201                                       OpenMPDirectiveKind Kind, bool EmitChecks,
3202                                       bool ForceSimpleCall) {
3203   if (!CGF.HaveInsertPoint())
3204     return;
3205   // Build call __kmpc_cancel_barrier(loc, thread_id);
3206   // Build call __kmpc_barrier(loc, thread_id);
3207   unsigned Flags;
3208   if (Kind == OMPD_for)
3209     Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3210   else if (Kind == OMPD_sections)
3211     Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3212   else if (Kind == OMPD_single)
3213     Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3214   else if (Kind == OMPD_barrier)
3215     Flags = OMP_IDENT_BARRIER_EXPL;
3216   else
3217     Flags = OMP_IDENT_BARRIER_IMPL;
3218   // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
3219   // thread_id);
3220   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
3221                          getThreadID(CGF, Loc)};
3222   if (auto *OMPRegionInfo =
3223           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
3224     if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3225       llvm::Value *Result = CGF.EmitRuntimeCall(
3226           createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
3227       if (EmitChecks) {
3228         // if (__kmpc_cancel_barrier()) {
3229         //   exit from construct;
3230         // }
3231         llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
3232         llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
3233         llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
3234         CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3235         CGF.EmitBlock(ExitBB);
3236         //   exit from construct;
3237         CodeGenFunction::JumpDest CancelDestination =
3238             CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
3239         CGF.EmitBranchThroughCleanup(CancelDestination);
3240         CGF.EmitBlock(ContBB, /*IsFinished=*/true);
3241       }
3242       return;
3243     }
3244   }
3245   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
3246 }
3247
3248 /// Map the OpenMP loop schedule to the runtime enumeration.
3249 static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
3250                                           bool Chunked, bool Ordered) {
3251   switch (ScheduleKind) {
3252   case OMPC_SCHEDULE_static:
3253     return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
3254                    : (Ordered ? OMP_ord_static : OMP_sch_static);
3255   case OMPC_SCHEDULE_dynamic:
3256     return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
3257   case OMPC_SCHEDULE_guided:
3258     return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
3259   case OMPC_SCHEDULE_runtime:
3260     return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
3261   case OMPC_SCHEDULE_auto:
3262     return Ordered ? OMP_ord_auto : OMP_sch_auto;
3263   case OMPC_SCHEDULE_unknown:
3264     assert(!Chunked && "chunk was specified but schedule kind not known");
3265     return Ordered ? OMP_ord_static : OMP_sch_static;
3266   }
3267   llvm_unreachable("Unexpected runtime schedule");
3268 }
3269
3270 /// Map the OpenMP distribute schedule to the runtime enumeration.
3271 static OpenMPSchedType
3272 getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
3273   // only static is allowed for dist_schedule
3274   return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
3275 }
3276
3277 bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
3278                                          bool Chunked) const {
3279   OpenMPSchedType Schedule =
3280       getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
3281   return Schedule == OMP_sch_static;
3282 }
3283
3284 bool CGOpenMPRuntime::isStaticNonchunked(
3285     OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
3286   OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
3287   return Schedule == OMP_dist_sch_static;
3288 }
3289
3290
3291 bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
3292   OpenMPSchedType Schedule =
3293       getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
3294   assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
3295   return Schedule != OMP_sch_static;
3296 }
3297
3298 static int addMonoNonMonoModifier(OpenMPSchedType Schedule,
3299                                   OpenMPScheduleClauseModifier M1,
3300                                   OpenMPScheduleClauseModifier M2) {
3301   int Modifier = 0;
3302   switch (M1) {
3303   case OMPC_SCHEDULE_MODIFIER_monotonic:
3304     Modifier = OMP_sch_modifier_monotonic;
3305     break;
3306   case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3307     Modifier = OMP_sch_modifier_nonmonotonic;
3308     break;
3309   case OMPC_SCHEDULE_MODIFIER_simd:
3310     if (Schedule == OMP_sch_static_chunked)
3311       Schedule = OMP_sch_static_balanced_chunked;
3312     break;
3313   case OMPC_SCHEDULE_MODIFIER_last:
3314   case OMPC_SCHEDULE_MODIFIER_unknown:
3315     break;
3316   }
3317   switch (M2) {
3318   case OMPC_SCHEDULE_MODIFIER_monotonic:
3319     Modifier = OMP_sch_modifier_monotonic;
3320     break;
3321   case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3322     Modifier = OMP_sch_modifier_nonmonotonic;
3323     break;
3324   case OMPC_SCHEDULE_MODIFIER_simd:
3325     if (Schedule == OMP_sch_static_chunked)
3326       Schedule = OMP_sch_static_balanced_chunked;
3327     break;
3328   case OMPC_SCHEDULE_MODIFIER_last:
3329   case OMPC_SCHEDULE_MODIFIER_unknown:
3330     break;
3331   }
3332   return Schedule | Modifier;
3333 }
3334
3335 void CGOpenMPRuntime::emitForDispatchInit(
3336     CodeGenFunction &CGF, SourceLocation Loc,
3337     const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
3338     bool Ordered, const DispatchRTInput &DispatchValues) {
3339   if (!CGF.HaveInsertPoint())
3340     return;
3341   OpenMPSchedType Schedule = getRuntimeSchedule(
3342       ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered);
3343   assert(Ordered ||
3344          (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
3345           Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
3346           Schedule != OMP_sch_static_balanced_chunked));
3347   // Call __kmpc_dispatch_init(
3348   //          ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
3349   //          kmp_int[32|64] lower, kmp_int[32|64] upper,
3350   //          kmp_int[32|64] stride, kmp_int[32|64] chunk);
3351
3352   // If the Chunk was not specified in the clause - use default value 1.
3353   llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
3354                                             : CGF.Builder.getIntN(IVSize, 1);
3355   llvm::Value *Args[] = {
3356       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3357       CGF.Builder.getInt32(addMonoNonMonoModifier(
3358           Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type
3359       DispatchValues.LB,                                // Lower
3360       DispatchValues.UB,                                // Upper
3361       CGF.Builder.getIntN(IVSize, 1),                   // Stride
3362       Chunk                                             // Chunk
3363   };
3364   CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
3365 }
3366
3367 static void emitForStaticInitCall(
3368     CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
3369     llvm::Constant *ForStaticInitFunction, OpenMPSchedType Schedule,
3370     OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
3371     const CGOpenMPRuntime::StaticRTInput &Values) {
3372   if (!CGF.HaveInsertPoint())
3373     return;
3374
3375   assert(!Values.Ordered);
3376   assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
3377          Schedule == OMP_sch_static_balanced_chunked ||
3378          Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
3379          Schedule == OMP_dist_sch_static ||
3380          Schedule == OMP_dist_sch_static_chunked);
3381
3382   // Call __kmpc_for_static_init(
3383   //          ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
3384   //          kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
3385   //          kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
3386   //          kmp_int[32|64] incr, kmp_int[32|64] chunk);
3387   llvm::Value *Chunk = Values.Chunk;
3388   if (Chunk == nullptr) {
3389     assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
3390             Schedule == OMP_dist_sch_static) &&
3391            "expected static non-chunked schedule");
3392     // If the Chunk was not specified in the clause - use default value 1.
3393     Chunk = CGF.Builder.getIntN(Values.IVSize, 1);
3394   } else {
3395     assert((Schedule == OMP_sch_static_chunked ||
3396             Schedule == OMP_sch_static_balanced_chunked ||
3397             Schedule == OMP_ord_static_chunked ||
3398             Schedule == OMP_dist_sch_static_chunked) &&
3399            "expected static chunked schedule");
3400   }
3401   llvm::Value *Args[] = {
3402       UpdateLocation,
3403       ThreadId,
3404       CGF.Builder.getInt32(addMonoNonMonoModifier(Schedule, M1,
3405                                                   M2)), // Schedule type
3406       Values.IL.getPointer(),                           // &isLastIter
3407       Values.LB.getPointer(),                           // &LB
3408       Values.UB.getPointer(),                           // &UB
3409       Values.ST.getPointer(),                           // &Stride
3410       CGF.Builder.getIntN(Values.IVSize, 1),            // Incr
3411       Chunk                                             // Chunk
3412   };
3413   CGF.EmitRuntimeCall(ForStaticInitFunction, Args);
3414 }
3415
3416 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
3417                                         SourceLocation Loc,
3418                                         OpenMPDirectiveKind DKind,
3419                                         const OpenMPScheduleTy &ScheduleKind,
3420                                         const StaticRTInput &Values) {
3421   OpenMPSchedType ScheduleNum = getRuntimeSchedule(
3422       ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered);
3423   assert(isOpenMPWorksharingDirective(DKind) &&
3424          "Expected loop-based or sections-based directive.");
3425   llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
3426                                              isOpenMPLoopDirective(DKind)
3427                                                  ? OMP_IDENT_WORK_LOOP
3428                                                  : OMP_IDENT_WORK_SECTIONS);
3429   llvm::Value *ThreadId = getThreadID(CGF, Loc);
3430   llvm::Constant *StaticInitFunction =
3431       createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3432   emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3433                         ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values);
3434 }
3435
3436 void CGOpenMPRuntime::emitDistributeStaticInit(
3437     CodeGenFunction &CGF, SourceLocation Loc,
3438     OpenMPDistScheduleClauseKind SchedKind,
3439     const CGOpenMPRuntime::StaticRTInput &Values) {
3440   OpenMPSchedType ScheduleNum =
3441       getRuntimeSchedule(SchedKind, Values.Chunk != nullptr);
3442   llvm::Value *UpdatedLocation =
3443       emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE);
3444   llvm::Value *ThreadId = getThreadID(CGF, Loc);
3445   llvm::Constant *StaticInitFunction =
3446       createForStaticInitFunction(Values.IVSize, Values.IVSigned);
3447   emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction,
3448                         ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown,
3449                         OMPC_SCHEDULE_MODIFIER_unknown, Values);
3450 }
3451
3452 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
3453                                           SourceLocation Loc,
3454                                           OpenMPDirectiveKind DKind) {
3455   if (!CGF.HaveInsertPoint())
3456     return;
3457   // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
3458   llvm::Value *Args[] = {
3459       emitUpdateLocation(CGF, Loc,
3460                          isOpenMPDistributeDirective(DKind)
3461                              ? OMP_IDENT_WORK_DISTRIBUTE
3462                              : isOpenMPLoopDirective(DKind)
3463                                    ? OMP_IDENT_WORK_LOOP
3464                                    : OMP_IDENT_WORK_SECTIONS),
3465       getThreadID(CGF, Loc)};
3466   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
3467                       Args);
3468 }
3469
3470 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
3471                                                  SourceLocation Loc,
3472                                                  unsigned IVSize,
3473                                                  bool IVSigned) {
3474   if (!CGF.HaveInsertPoint())
3475     return;
3476   // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
3477   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
3478   CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
3479 }
3480
3481 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
3482                                           SourceLocation Loc, unsigned IVSize,
3483                                           bool IVSigned, Address IL,
3484                                           Address LB, Address UB,
3485                                           Address ST) {
3486   // Call __kmpc_dispatch_next(
3487   //          ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
3488   //          kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
3489   //          kmp_int[32|64] *p_stride);
3490   llvm::Value *Args[] = {
3491       emitUpdateLocation(CGF, Loc),
3492       getThreadID(CGF, Loc),
3493       IL.getPointer(), // &isLastIter
3494       LB.getPointer(), // &Lower
3495       UB.getPointer(), // &Upper
3496       ST.getPointer()  // &Stride
3497   };
3498   llvm::Value *Call =
3499       CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
3500   return CGF.EmitScalarConversion(
3501       Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1),
3502       CGF.getContext().BoolTy, Loc);
3503 }
3504
3505 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
3506                                            llvm::Value *NumThreads,
3507                                            SourceLocation Loc) {
3508   if (!CGF.HaveInsertPoint())
3509     return;
3510   // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
3511   llvm::Value *Args[] = {
3512       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3513       CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
3514   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
3515                       Args);
3516 }
3517
3518 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
3519                                          OpenMPProcBindClauseKind ProcBind,
3520                                          SourceLocation Loc) {
3521   if (!CGF.HaveInsertPoint())
3522     return;
3523   // Constants for proc bind value accepted by the runtime.
3524   enum ProcBindTy {
3525     ProcBindFalse = 0,
3526     ProcBindTrue,
3527     ProcBindMaster,
3528     ProcBindClose,
3529     ProcBindSpread,
3530     ProcBindIntel,
3531     ProcBindDefault
3532   } RuntimeProcBind;
3533   switch (ProcBind) {
3534   case OMPC_PROC_BIND_master:
3535     RuntimeProcBind = ProcBindMaster;
3536     break;
3537   case OMPC_PROC_BIND_close:
3538     RuntimeProcBind = ProcBindClose;
3539     break;
3540   case OMPC_PROC_BIND_spread:
3541     RuntimeProcBind = ProcBindSpread;
3542     break;
3543   case OMPC_PROC_BIND_unknown:
3544     llvm_unreachable("Unsupported proc_bind value.");
3545   }
3546   // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
3547   llvm::Value *Args[] = {
3548       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
3549       llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
3550   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
3551 }
3552
3553 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
3554                                 SourceLocation Loc) {
3555   if (!CGF.HaveInsertPoint())
3556     return;
3557   // Build call void __kmpc_flush(ident_t *loc)
3558   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
3559                       emitUpdateLocation(CGF, Loc));
3560 }
3561
3562 namespace {
3563 /// Indexes of fields for type kmp_task_t.
3564 enum KmpTaskTFields {
3565   /// List of shared variables.
3566   KmpTaskTShareds,
3567   /// Task routine.
3568   KmpTaskTRoutine,
3569   /// Partition id for the untied tasks.
3570   KmpTaskTPartId,
3571   /// Function with call of destructors for private variables.
3572   Data1,
3573   /// Task priority.
3574   Data2,
3575   /// (Taskloops only) Lower bound.
3576   KmpTaskTLowerBound,
3577   /// (Taskloops only) Upper bound.
3578   KmpTaskTUpperBound,
3579   /// (Taskloops only) Stride.
3580   KmpTaskTStride,
3581   /// (Taskloops only) Is last iteration flag.
3582   KmpTaskTLastIter,
3583   /// (Taskloops only) Reduction data.
3584   KmpTaskTReductions,
3585 };
3586 } // anonymous namespace
3587
3588 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
3589   return OffloadEntriesTargetRegion.empty() &&
3590          OffloadEntriesDeviceGlobalVar.empty();
3591 }
3592
3593 /// Initialize target region entry.
3594 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3595     initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3596                                     StringRef ParentName, unsigned LineNum,
3597                                     unsigned Order) {
3598   assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3599                                              "only required for the device "
3600                                              "code generation.");
3601   OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3602       OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
3603                                    OMPTargetRegionEntryTargetRegion);
3604   ++OffloadingEntriesNum;
3605 }
3606
3607 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3608     registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID,
3609                                   StringRef ParentName, unsigned LineNum,
3610                                   llvm::Constant *Addr, llvm::Constant *ID,
3611                                   OMPTargetRegionEntryKind Flags) {
3612   // If we are emitting code for a target, the entry is already initialized,
3613   // only has to be registered.
3614   if (CGM.getLangOpts().OpenMPIsDevice) {
3615     if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3616       unsigned DiagID = CGM.getDiags().getCustomDiagID(
3617           DiagnosticsEngine::Error,
3618           "Unable to find target region on line '%0' in the device code.");
3619       CGM.getDiags().Report(DiagID) << LineNum;
3620       return;
3621     }
3622     auto &Entry =
3623         OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3624     assert(Entry.isValid() && "Entry not initialized!");
3625     Entry.setAddress(Addr);
3626     Entry.setID(ID);
3627     Entry.setFlags(Flags);
3628   } else {
3629     OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3630     OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3631     ++OffloadingEntriesNum;
3632   }
3633 }
3634
3635 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3636     unsigned DeviceID, unsigned FileID, StringRef ParentName,
3637     unsigned LineNum) const {
3638   auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3639   if (PerDevice == OffloadEntriesTargetRegion.end())
3640     return false;
3641   auto PerFile = PerDevice->second.find(FileID);
3642   if (PerFile == PerDevice->second.end())
3643     return false;
3644   auto PerParentName = PerFile->second.find(ParentName);
3645   if (PerParentName == PerFile->second.end())
3646     return false;
3647   auto PerLine = PerParentName->second.find(LineNum);
3648   if (PerLine == PerParentName->second.end())
3649     return false;
3650   // Fail if this entry is already registered.
3651   if (PerLine->second.getAddress() || PerLine->second.getID())
3652     return false;
3653   return true;
3654 }
3655
3656 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3657     const OffloadTargetRegionEntryInfoActTy &Action) {
3658   // Scan all target region entries and perform the provided action.
3659   for (const auto &D : OffloadEntriesTargetRegion)
3660     for (const auto &F : D.second)
3661       for (const auto &P : F.second)
3662         for (const auto &L : P.second)
3663           Action(D.first, F.first, P.first(), L.first, L.second);
3664 }
3665
3666 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3667     initializeDeviceGlobalVarEntryInfo(StringRef Name,
3668                                        OMPTargetGlobalVarEntryKind Flags,
3669                                        unsigned Order) {
3670   assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
3671                                              "only required for the device "
3672                                              "code generation.");
3673   OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3674   ++OffloadingEntriesNum;
3675 }
3676
3677 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3678     registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
3679                                      CharUnits VarSize,
3680                                      OMPTargetGlobalVarEntryKind Flags,
3681                                      llvm::GlobalValue::LinkageTypes Linkage) {
3682   if (CGM.getLangOpts().OpenMPIsDevice) {
3683     auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
3684     assert(Entry.isValid() && Entry.getFlags() == Flags &&
3685            "Entry not initialized!");
3686     assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
3687            "Resetting with the new address.");
3688     if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName))
3689       return;
3690     Entry.setAddress(Addr);
3691     Entry.setVarSize(VarSize);
3692     Entry.setLinkage(Linkage);
3693   } else {
3694     if (hasDeviceGlobalVarEntryInfo(VarName))
3695       return;
3696     OffloadEntriesDeviceGlobalVar.try_emplace(
3697         VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
3698     ++OffloadingEntriesNum;
3699   }
3700 }
3701
3702 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3703     actOnDeviceGlobalVarEntriesInfo(
3704         const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
3705   // Scan all target region entries and perform the provided action.
3706   for (const auto &E : OffloadEntriesDeviceGlobalVar)
3707     Action(E.getKey(), E.getValue());
3708 }
3709
3710 llvm::Function *
3711 CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
3712   // If we don't have entries or if we are emitting code for the device, we
3713   // don't need to do anything.
3714   if (CGM.getLangOpts().OpenMPIsDevice || OffloadEntriesInfoManager.empty())
3715     return nullptr;
3716
3717   llvm::Module &M = CGM.getModule();
3718   ASTContext &C = CGM.getContext();
3719
3720   // Get list of devices we care about
3721   const std::vector<llvm::Triple> &Devices = CGM.getLangOpts().OMPTargetTriples;
3722
3723   // We should be creating an offloading descriptor only if there are devices
3724   // specified.
3725   assert(!Devices.empty() && "No OpenMP offloading devices??");
3726
3727   // Create the external variables that will point to the begin and end of the
3728   // host entries section. These will be defined by the linker.
3729   llvm::Type *OffloadEntryTy =
3730       CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
3731   std::string EntriesBeginName = getName({"omp_offloading", "entries_begin"});
3732   auto *HostEntriesBegin = new llvm::GlobalVariable(
3733       M, OffloadEntryTy, /*isConstant=*/true,
3734       llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
3735       EntriesBeginName);
3736   std::string EntriesEndName = getName({"omp_offloading", "entries_end"});
3737   auto *HostEntriesEnd =
3738       new llvm::GlobalVariable(M, OffloadEntryTy, /*isConstant=*/true,
3739                                llvm::GlobalValue::ExternalLinkage,
3740                                /*Initializer=*/nullptr, EntriesEndName);
3741
3742   // Create all device images
3743   auto *DeviceImageTy = cast<llvm::StructType>(
3744       CGM.getTypes().ConvertTypeForMem(getTgtDeviceImageQTy()));
3745   ConstantInitBuilder DeviceImagesBuilder(CGM);
3746   ConstantArrayBuilder DeviceImagesEntries =
3747       DeviceImagesBuilder.beginArray(DeviceImageTy);
3748
3749   for (const llvm::Triple &Device : Devices) {
3750     StringRef T = Device.getTriple();
3751     std::string BeginName = getName({"omp_offloading", "img_start", ""});
3752     auto *ImgBegin = new llvm::GlobalVariable(
3753         M, CGM.Int8Ty, /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage,
3754         /*Initializer=*/nullptr, Twine(BeginName).concat(T));
3755     std::string EndName = getName({"omp_offloading", "img_end", ""});
3756     auto *ImgEnd = new llvm::GlobalVariable(
3757         M, CGM.Int8Ty, /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage,
3758         /*Initializer=*/nullptr, Twine(EndName).concat(T));
3759
3760     llvm::Constant *Data[] = {ImgBegin, ImgEnd, HostEntriesBegin,
3761                               HostEntriesEnd};
3762     createConstantGlobalStructAndAddToParent(CGM, getTgtDeviceImageQTy(), Data,
3763                                              DeviceImagesEntries);
3764   }
3765
3766   // Create device images global array.
3767   std::string ImagesName = getName({"omp_offloading", "device_images"});
3768   llvm::GlobalVariable *DeviceImages =
3769       DeviceImagesEntries.finishAndCreateGlobal(ImagesName,
3770                                                 CGM.getPointerAlign(),
3771                                                 /*isConstant=*/true);
3772   DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3773
3774   // This is a Zero array to be used in the creation of the constant expressions
3775   llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
3776                              llvm::Constant::getNullValue(CGM.Int32Ty)};
3777
3778   // Create the target region descriptor.
3779   llvm::Constant *Data[] = {
3780       llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
3781       llvm::ConstantExpr::getGetElementPtr(DeviceImages->getValueType(),
3782                                            DeviceImages, Index),
3783       HostEntriesBegin, HostEntriesEnd};
3784   std::string Descriptor = getName({"omp_offloading", "descriptor"});
3785   llvm::GlobalVariable *Desc = createConstantGlobalStruct(
3786       CGM, getTgtBinaryDescriptorQTy(), Data, Descriptor);
3787
3788   // Emit code to register or unregister the descriptor at execution
3789   // startup or closing, respectively.
3790
3791   llvm::Function *UnRegFn;
3792   {
3793     FunctionArgList Args;
3794     ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
3795     Args.push_back(&DummyPtr);
3796
3797     CodeGenFunction CGF(CGM);
3798     // Disable debug info for global (de-)initializer because they are not part
3799     // of some particular construct.
3800     CGF.disableDebugInfo();
3801     const auto &FI =
3802         CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
3803     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3804     std::string UnregName = getName({"omp_offloading", "descriptor_unreg"});
3805     UnRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, UnregName, FI);
3806     CGF.StartFunction(GlobalDecl(), C.VoidTy, UnRegFn, FI, Args);
3807     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
3808                         Desc);
3809     CGF.FinishFunction();
3810   }
3811   llvm::Function *RegFn;
3812   {
3813     CodeGenFunction CGF(CGM);
3814     // Disable debug info for global (de-)initializer because they are not part
3815     // of some particular construct.
3816     CGF.disableDebugInfo();
3817     const auto &FI = CGM.getTypes().arrangeNullaryFunction();
3818     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
3819     std::string Descriptor = getName({"omp_offloading", "descriptor_reg"});
3820     RegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, Descriptor, FI);
3821     CGF.StartFunction(GlobalDecl(), C.VoidTy, RegFn, FI, FunctionArgList());
3822     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc);
3823     // Create a variable to drive the registration and unregistration of the
3824     // descriptor, so we can reuse the logic that emits Ctors and Dtors.
3825     ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(),
3826                                   SourceLocation(), nullptr, C.CharTy,
3827                                   ImplicitParamDecl::Other);
3828     CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
3829     CGF.FinishFunction();
3830   }
3831   if (CGM.supportsCOMDAT()) {
3832     // It is sufficient to call registration function only once, so create a
3833     // COMDAT group for registration/unregistration functions and associated
3834     // data. That would reduce startup time and code size. Registration
3835     // function serves as a COMDAT group key.
3836     llvm::Comdat *ComdatKey = M.getOrInsertComdat(RegFn->getName());
3837     RegFn->setLinkage(llvm::GlobalValue::LinkOnceAnyLinkage);
3838     RegFn->setVisibility(llvm::GlobalValue::HiddenVisibility);
3839     RegFn->setComdat(ComdatKey);
3840     UnRegFn->setComdat(ComdatKey);
3841     DeviceImages->setComdat(ComdatKey);
3842     Desc->setComdat(ComdatKey);
3843   }
3844   return RegFn;
3845 }
3846
3847 void CGOpenMPRuntime::createOffloadEntry(
3848     llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
3849     llvm::GlobalValue::LinkageTypes Linkage) {
3850   StringRef Name = Addr->getName();
3851   llvm::Module &M = CGM.getModule();
3852   llvm::LLVMContext &C = M.getContext();
3853
3854   // Create constant string with the name.
3855   llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
3856
3857   std::string StringName = getName({"omp_offloading", "entry_name"});
3858   auto *Str = new llvm::GlobalVariable(
3859       M, StrPtrInit->getType(), /*isConstant=*/true,
3860       llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName);
3861   Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3862
3863   llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy),
3864                             llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy),
3865                             llvm::ConstantInt::get(CGM.SizeTy, Size),
3866                             llvm::ConstantInt::get(CGM.Int32Ty, Flags),
3867                             llvm::ConstantInt::get(CGM.Int32Ty, 0)};
3868   std::string EntryName = getName({"omp_offloading", "entry", ""});
3869   llvm::GlobalVariable *Entry = createConstantGlobalStruct(
3870       CGM, getTgtOffloadEntryQTy(), Data, Twine(EntryName).concat(Name),
3871       llvm::GlobalValue::WeakAnyLinkage);
3872
3873   // The entry has to be created in the section the linker expects it to be.
3874   std::string Section = getName({"omp_offloading", "entries"});
3875   Entry->setSection(Section);
3876 }
3877
3878 void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
3879   // Emit the offloading entries and metadata so that the device codegen side
3880   // can easily figure out what to emit. The produced metadata looks like
3881   // this:
3882   //
3883   // !omp_offload.info = !{!1, ...}
3884   //
3885   // Right now we only generate metadata for function that contain target
3886   // regions.
3887
3888   // If we do not have entries, we don't need to do anything.
3889   if (OffloadEntriesInfoManager.empty())
3890     return;
3891
3892   llvm::Module &M = CGM.getModule();
3893   llvm::LLVMContext &C = M.getContext();
3894   SmallVector<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
3895       OrderedEntries(OffloadEntriesInfoManager.size());
3896
3897   // Auxiliary methods to create metadata values and strings.
3898   auto &&GetMDInt = [this](unsigned V) {
3899     return llvm::ConstantAsMetadata::get(
3900         llvm::ConstantInt::get(CGM.Int32Ty, V));
3901   };
3902
3903   auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); };
3904
3905   // Create the offloading info metadata node.
3906   llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
3907
3908   // Create function that emits metadata for each target region entry;
3909   auto &&TargetRegionMetadataEmitter =
3910       [&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
3911           unsigned DeviceID, unsigned FileID, StringRef ParentName,
3912           unsigned Line,
3913           const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
3914         // Generate metadata for target regions. Each entry of this metadata
3915         // contains:
3916         // - Entry 0 -> Kind of this type of metadata (0).
3917         // - Entry 1 -> Device ID of the file where the entry was identified.
3918         // - Entry 2 -> File ID of the file where the entry was identified.
3919         // - Entry 3 -> Mangled name of the function where the entry was
3920         // identified.
3921         // - Entry 4 -> Line in the file where the entry was identified.
3922         // - Entry 5 -> Order the entry was created.
3923         // The first element of the metadata node is the kind.
3924         llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
3925                                  GetMDInt(FileID),      GetMDString(ParentName),
3926                                  GetMDInt(Line),        GetMDInt(E.getOrder())};
3927
3928         // Save this entry in the right position of the ordered entries array.
3929         OrderedEntries[E.getOrder()] = &E;
3930
3931         // Add metadata to the named metadata node.
3932         MD->addOperand(llvm::MDNode::get(C, Ops));
3933       };
3934
3935   OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
3936       TargetRegionMetadataEmitter);
3937
3938   // Create function that emits metadata for each device global variable entry;
3939   auto &&DeviceGlobalVarMetadataEmitter =
3940       [&C, &OrderedEntries, &GetMDInt, &GetMDString,
3941        MD](StringRef MangledName,
3942            const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar
3943                &E) {
3944         // Generate metadata for global variables. Each entry of this metadata
3945         // contains:
3946         // - Entry 0 -> Kind of this type of metadata (1).
3947         // - Entry 1 -> Mangled name of the variable.
3948         // - Entry 2 -> Declare target kind.
3949         // - Entry 3 -> Order the entry was created.
3950         // The first element of the metadata node is the kind.
3951         llvm::Metadata *Ops[] = {
3952             GetMDInt(E.getKind()), GetMDString(MangledName),
3953             GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
3954
3955         // Save this entry in the right position of the ordered entries array.
3956         OrderedEntries[E.getOrder()] = &E;
3957
3958         // Add metadata to the named metadata node.
3959         MD->addOperand(llvm::MDNode::get(C, Ops));
3960       };
3961
3962   OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
3963       DeviceGlobalVarMetadataEmitter);
3964
3965   for (const auto *E : OrderedEntries) {
3966     assert(E && "All ordered entries must exist!");
3967     if (const auto *CE =
3968             dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
3969                 E)) {
3970       if (!CE->getID() || !CE->getAddress()) {
3971         unsigned DiagID = CGM.getDiags().getCustomDiagID(
3972             DiagnosticsEngine::Error,
3973             "Offloading entry for target region is incorrect: either the "
3974             "address or the ID is invalid.");
3975         CGM.getDiags().Report(DiagID);
3976         continue;
3977       }
3978       createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
3979                          CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
3980     } else if (const auto *CE =
3981                    dyn_cast<OffloadEntriesInfoManagerTy::
3982                                 OffloadEntryInfoDeviceGlobalVar>(E)) {
3983       OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags =
3984           static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
3985               CE->getFlags());
3986       switch (Flags) {
3987       case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: {
3988         if (!CE->getAddress()) {
3989           unsigned DiagID = CGM.getDiags().getCustomDiagID(
3990               DiagnosticsEngine::Error,
3991               "Offloading entry for declare target variable is incorrect: the "
3992               "address is invalid.");
3993           CGM.getDiags().Report(DiagID);
3994           continue;
3995         }
3996         break;
3997       }
3998       case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink:
3999         assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
4000                 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) &&
4001                "Declaret target link address is set.");
4002         if (CGM.getLangOpts().OpenMPIsDevice)
4003           continue;
4004         if (!CE->getAddress()) {
4005           unsigned DiagID = CGM.getDiags().getCustomDiagID(
4006               DiagnosticsEngine::Error,
4007               "Offloading entry for declare target variable is incorrect: the "
4008               "address is invalid.");
4009           CGM.getDiags().Report(DiagID);
4010           continue;
4011         }
4012         break;
4013       }
4014       createOffloadEntry(CE->getAddress(), CE->getAddress(),
4015                          CE->getVarSize().getQuantity(), Flags,
4016                          CE->getLinkage());
4017     } else {
4018       llvm_unreachable("Unsupported entry kind.");
4019     }
4020   }
4021 }
4022
4023 /// Loads all the offload entries information from the host IR
4024 /// metadata.
4025 void CGOpenMPRuntime::loadOffloadInfoMetadata() {
4026   // If we are in target mode, load the metadata from the host IR. This code has
4027   // to match the metadaata creation in createOffloadEntriesAndInfoMetadata().
4028
4029   if (!CGM.getLangOpts().OpenMPIsDevice)
4030     return;
4031
4032   if (CGM.getLangOpts().OMPHostIRFile.empty())
4033     return;
4034
4035   auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
4036   if (auto EC = Buf.getError()) {
4037     CGM.getDiags().Report(diag::err_cannot_open_file)
4038         << CGM.getLangOpts().OMPHostIRFile << EC.message();
4039     return;
4040   }
4041
4042   llvm::LLVMContext C;
4043   auto ME = expectedToErrorOrAndEmitErrors(
4044       C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4045
4046   if (auto EC = ME.getError()) {
4047     unsigned DiagID = CGM.getDiags().getCustomDiagID(
4048         DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'");
4049     CGM.getDiags().Report(DiagID)
4050         << CGM.getLangOpts().OMPHostIRFile << EC.message();
4051     return;
4052   }
4053
4054   llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info");
4055   if (!MD)
4056     return;
4057
4058   for (llvm::MDNode *MN : MD->operands()) {
4059     auto &&GetMDInt = [MN](unsigned Idx) {
4060       auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4061       return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4062     };
4063
4064     auto &&GetMDString = [MN](unsigned Idx) {
4065       auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4066       return V->getString();
4067     };
4068
4069     switch (GetMDInt(0)) {
4070     default:
4071       llvm_unreachable("Unexpected metadata!");
4072       break;
4073     case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4074         OffloadingEntryInfoTargetRegion:
4075       OffloadEntriesInfoManager.initializeTargetRegionEntryInfo(
4076           /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2),
4077           /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4),
4078           /*Order=*/GetMDInt(5));
4079       break;
4080     case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
4081         OffloadingEntryInfoDeviceGlobalVar:
4082       OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
4083           /*MangledName=*/GetMDString(1),
4084           static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4085               /*Flags=*/GetMDInt(2)),
4086           /*Order=*/GetMDInt(3));
4087       break;
4088     }
4089   }
4090 }
4091
4092 void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
4093   if (!KmpRoutineEntryPtrTy) {
4094     // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
4095     ASTContext &C = CGM.getContext();
4096     QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
4097     FunctionProtoType::ExtProtoInfo EPI;
4098     KmpRoutineEntryPtrQTy = C.getPointerType(
4099         C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
4100     KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
4101   }
4102 }
4103
4104 QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() {
4105   // Make sure the type of the entry is already created. This is the type we
4106   // have to create:
4107   // struct __tgt_offload_entry{
4108   //   void      *addr;       // Pointer to the offload entry info.
4109   //                          // (function or global)
4110   //   char      *name;       // Name of the function or global.
4111   //   size_t     size;       // Size of the entry info (0 if it a function).
4112   //   int32_t    flags;      // Flags associated with the entry, e.g. 'link'.
4113   //   int32_t    reserved;   // Reserved, to use by the runtime library.
4114   // };
4115   if (TgtOffloadEntryQTy.isNull()) {
4116     ASTContext &C = CGM.getContext();
4117     RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry");
4118     RD->startDefinition();
4119     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4120     addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy));
4121     addFieldToRecordDecl(C, RD, C.getSizeType());
4122     addFieldToRecordDecl(
4123         C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4124     addFieldToRecordDecl(
4125         C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4126     RD->completeDefinition();
4127     RD->addAttr(PackedAttr::CreateImplicit(C));
4128     TgtOffloadEntryQTy = C.getRecordType(RD);
4129   }
4130   return TgtOffloadEntryQTy;
4131 }
4132
4133 QualType CGOpenMPRuntime::getTgtDeviceImageQTy() {
4134   // These are the types we need to build:
4135   // struct __tgt_device_image{
4136   // void   *ImageStart;       // Pointer to the target code start.
4137   // void   *ImageEnd;         // Pointer to the target code end.
4138   // // We also add the host entries to the device image, as it may be useful
4139   // // for the target runtime to have access to that information.
4140   // __tgt_offload_entry  *EntriesBegin;   // Begin of the table with all
4141   //                                       // the entries.
4142   // __tgt_offload_entry  *EntriesEnd;     // End of the table with all the
4143   //                                       // entries (non inclusive).
4144   // };
4145   if (TgtDeviceImageQTy.isNull()) {
4146     ASTContext &C = CGM.getContext();
4147     RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image");
4148     RD->startDefinition();
4149     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4150     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4151     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4152     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4153     RD->completeDefinition();
4154     TgtDeviceImageQTy = C.getRecordType(RD);
4155   }
4156   return TgtDeviceImageQTy;
4157 }
4158
4159 QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() {
4160   // struct __tgt_bin_desc{
4161   //   int32_t              NumDevices;      // Number of devices supported.
4162   //   __tgt_device_image   *DeviceImages;   // Arrays of device images
4163   //                                         // (one per device).
4164   //   __tgt_offload_entry  *EntriesBegin;   // Begin of the table with all the
4165   //                                         // entries.
4166   //   __tgt_offload_entry  *EntriesEnd;     // End of the table with all the
4167   //                                         // entries (non inclusive).
4168   // };
4169   if (TgtBinaryDescriptorQTy.isNull()) {
4170     ASTContext &C = CGM.getContext();
4171     RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc");
4172     RD->startDefinition();
4173     addFieldToRecordDecl(
4174         C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true));
4175     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy()));
4176     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4177     addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy()));
4178     RD->completeDefinition();
4179     TgtBinaryDescriptorQTy = C.getRecordType(RD);
4180   }
4181   return TgtBinaryDescriptorQTy;
4182 }
4183
4184 namespace {
4185 struct PrivateHelpersTy {
4186   PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
4187                    const VarDecl *PrivateElemInit)
4188       : Original(Original), PrivateCopy(PrivateCopy),
4189         PrivateElemInit(PrivateElemInit) {}
4190   const VarDecl *Original;
4191   const VarDecl *PrivateCopy;
4192   const VarDecl *PrivateElemInit;
4193 };
4194 typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
4195 } // anonymous namespace
4196
4197 static RecordDecl *
4198 createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
4199   if (!Privates.empty()) {
4200     ASTContext &C = CGM.getContext();
4201     // Build struct .kmp_privates_t. {
4202     //         /*  private vars  */
4203     //       };
4204     RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t");
4205     RD->startDefinition();
4206     for (const auto &Pair : Privates) {
4207       const VarDecl *VD = Pair.second.Original;
4208       QualType Type = VD->getType().getNonReferenceType();
4209       FieldDecl *FD = addFieldToRecordDecl(C, RD, Type);
4210       if (VD->hasAttrs()) {
4211         for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
4212              E(VD->getAttrs().end());
4213              I != E; ++I)
4214           FD->addAttr(*I);
4215       }
4216     }
4217     RD->completeDefinition();
4218     return RD;
4219   }
4220   return nullptr;
4221 }
4222
4223 static RecordDecl *
4224 createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
4225                          QualType KmpInt32Ty,
4226                          QualType KmpRoutineEntryPointerQTy) {
4227   ASTContext &C = CGM.getContext();
4228   // Build struct kmp_task_t {
4229   //         void *              shareds;
4230   //         kmp_routine_entry_t routine;
4231   //         kmp_int32           part_id;
4232   //         kmp_cmplrdata_t data1;
4233   //         kmp_cmplrdata_t data2;
4234   // For taskloops additional fields:
4235   //         kmp_uint64          lb;
4236   //         kmp_uint64          ub;
4237   //         kmp_int64           st;
4238   //         kmp_int32           liter;
4239   //         void *              reductions;
4240   //       };
4241   RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union);
4242   UD->startDefinition();
4243   addFieldToRecordDecl(C, UD, KmpInt32Ty);
4244   addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
4245   UD->completeDefinition();
4246   QualType KmpCmplrdataTy = C.getRecordType(UD);
4247   RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
4248   RD->startDefinition();
4249   addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4250   addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
4251   addFieldToRecordDecl(C, RD, KmpInt32Ty);
4252   addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4253   addFieldToRecordDecl(C, RD, KmpCmplrdataTy);
4254   if (isOpenMPTaskLoopDirective(Kind)) {
4255     QualType KmpUInt64Ty =
4256         CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
4257     QualType KmpInt64Ty =
4258         CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
4259     addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4260     addFieldToRecordDecl(C, RD, KmpUInt64Ty);
4261     addFieldToRecordDecl(C, RD, KmpInt64Ty);
4262     addFieldToRecordDecl(C, RD, KmpInt32Ty);
4263     addFieldToRecordDecl(C, RD, C.VoidPtrTy);
4264   }
4265   RD->completeDefinition();
4266   return RD;
4267 }
4268
4269 static RecordDecl *
4270 createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
4271                                      ArrayRef<PrivateDataTy> Privates) {
4272   ASTContext &C = CGM.getContext();
4273   // Build struct kmp_task_t_with_privates {
4274   //         kmp_task_t task_data;
4275   //         .kmp_privates_t. privates;
4276   //       };
4277   RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
4278   RD->startDefinition();
4279   addFieldToRecordDecl(C, RD, KmpTaskTQTy);
4280   if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
4281     addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
4282   RD->completeDefinition();
4283   return RD;
4284 }
4285
4286 /// Emit a proxy function which accepts kmp_task_t as the second
4287 /// argument.
4288 /// \code
4289 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
4290 ///   TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
4291 ///   For taskloops:
4292 ///   tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4293 ///   tt->reductions, tt->shareds);
4294 ///   return 0;
4295 /// }
4296 /// \endcode
4297 static llvm::Value *
4298 emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
4299                       OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
4300                       QualType KmpTaskTWithPrivatesPtrQTy,
4301                       QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
4302                       QualType SharedsPtrTy, llvm::Value *TaskFunction,
4303                       llvm::Value *TaskPrivatesMap) {
4304   ASTContext &C = CGM.getContext();
4305   FunctionArgList Args;
4306   ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4307                             ImplicitParamDecl::Other);
4308   ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4309                                 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4310                                 ImplicitParamDecl::Other);
4311   Args.push_back(&GtidArg);
4312   Args.push_back(&TaskTypeArg);
4313   const auto &TaskEntryFnInfo =
4314       CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4315   llvm::FunctionType *TaskEntryTy =
4316       CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
4317   std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""});
4318   auto *TaskEntry = llvm::Function::Create(
4319       TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4320   CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo);
4321   TaskEntry->setDoesNotRecurse();
4322   CodeGenFunction CGF(CGM);
4323   CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args,
4324                     Loc, Loc);
4325
4326   // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
4327   // tt,
4328   // For taskloops:
4329   // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
4330   // tt->task_data.shareds);
4331   llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
4332       CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc);
4333   LValue TDBase = CGF.EmitLoadOfPointerLValue(
4334       CGF.GetAddrOfLocalVar(&TaskTypeArg),
4335       KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4336   const auto *KmpTaskTWithPrivatesQTyRD =
4337       cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4338   LValue Base =
4339       CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4340   const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4341   auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4342   LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
4343   llvm::Value *PartidParam = PartIdLVal.getPointer();
4344
4345   auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4346   LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
4347   llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4348       CGF.EmitLoadOfScalar(SharedsLVal, Loc),
4349       CGF.ConvertTypeForMem(SharedsPtrTy));
4350
4351   auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4352   llvm::Value *PrivatesParam;
4353   if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4354     LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
4355     PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4356         PrivatesLVal.getPointer(), CGF.VoidPtrTy);
4357   } else {
4358     PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4359   }
4360
4361   llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4362                                TaskPrivatesMap,
4363                                CGF.Builder
4364                                    .CreatePointerBitCastOrAddrSpaceCast(
4365                                        TDBase.getAddress(), CGF.VoidPtrTy)
4366                                    .getPointer()};
4367   SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
4368                                           std::end(CommonArgs));
4369   if (isOpenMPTaskLoopDirective(Kind)) {
4370     auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4371     LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI);
4372     llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc);
4373     auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4374     LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI);
4375     llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc);
4376     auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4377     LValue StLVal = CGF.EmitLValueForField(Base, *StFI);
4378     llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc);
4379     auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4380     LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4381     llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc);
4382     auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4383     LValue RLVal = CGF.EmitLValueForField(Base, *RFI);
4384     llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc);
4385     CallArgs.push_back(LBParam);
4386     CallArgs.push_back(UBParam);
4387     CallArgs.push_back(StParam);
4388     CallArgs.push_back(LIParam);
4389     CallArgs.push_back(RParam);
4390   }
4391   CallArgs.push_back(SharedsParam);
4392
4393   CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction,
4394                                                   CallArgs);
4395   CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)),
4396                              CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty));
4397   CGF.FinishFunction();
4398   return TaskEntry;
4399 }
4400
4401 static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
4402                                             SourceLocation Loc,
4403                                             QualType KmpInt32Ty,
4404                                             QualType KmpTaskTWithPrivatesPtrQTy,
4405                                             QualType KmpTaskTWithPrivatesQTy) {
4406   ASTContext &C = CGM.getContext();
4407   FunctionArgList Args;
4408   ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty,
4409                             ImplicitParamDecl::Other);
4410   ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4411                                 KmpTaskTWithPrivatesPtrQTy.withRestrict(),
4412                                 ImplicitParamDecl::Other);
4413   Args.push_back(&GtidArg);
4414   Args.push_back(&TaskTypeArg);
4415   const auto &DestructorFnInfo =
4416       CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
4417   llvm::FunctionType *DestructorFnTy =
4418       CGM.getTypes().GetFunctionType(DestructorFnInfo);
4419   std::string Name =
4420       CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""});
4421   auto *DestructorFn =
4422       llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
4423                              Name, &CGM.getModule());
4424   CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn,
4425                                     DestructorFnInfo);
4426   DestructorFn->setDoesNotRecurse();
4427   CodeGenFunction CGF(CGM);
4428   CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
4429                     Args, Loc, Loc);
4430
4431   LValue Base = CGF.EmitLoadOfPointerLValue(
4432       CGF.GetAddrOfLocalVar(&TaskTypeArg),
4433       KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4434   const auto *KmpTaskTWithPrivatesQTyRD =
4435       cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
4436   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4437   Base = CGF.EmitLValueForField(Base, *FI);
4438   for (const auto *Field :
4439        cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4440     if (QualType::DestructionKind DtorKind =
4441             Field->getType().isDestructedType()) {
4442       LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
4443       CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
4444     }
4445   }
4446   CGF.FinishFunction();
4447   return DestructorFn;
4448 }
4449
4450 /// Emit a privates mapping function for correct handling of private and
4451 /// firstprivate variables.
4452 /// \code
4453 /// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
4454 /// **noalias priv1,...,  <tyn> **noalias privn) {
4455 ///   *priv1 = &.privates.priv1;
4456 ///   ...;
4457 ///   *privn = &.privates.privn;
4458 /// }
4459 /// \endcode
4460 static llvm::Value *
4461 emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
4462                                ArrayRef<const Expr *> PrivateVars,
4463                                ArrayRef<const Expr *> FirstprivateVars,
4464                                ArrayRef<const Expr *> LastprivateVars,
4465                                QualType PrivatesQTy,
4466                                ArrayRef<PrivateDataTy> Privates) {
4467   ASTContext &C = CGM.getContext();
4468   FunctionArgList Args;
4469   ImplicitParamDecl TaskPrivatesArg(
4470       C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4471       C.getPointerType(PrivatesQTy).withConst().withRestrict(),
4472       ImplicitParamDecl::Other);
4473   Args.push_back(&TaskPrivatesArg);
4474   llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4475   unsigned Counter = 1;
4476   for (const Expr *E : PrivateVars) {
4477     Args.push_back(ImplicitParamDecl::Create(
4478         C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4479         C.getPointerType(C.getPointerType(E->getType()))
4480             .withConst()
4481             .withRestrict(),
4482         ImplicitParamDecl::Other));
4483     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4484     PrivateVarsPos[VD] = Counter;
4485     ++Counter;
4486   }
4487   for (const Expr *E : FirstprivateVars) {
4488     Args.push_back(ImplicitParamDecl::Create(
4489         C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4490         C.getPointerType(C.getPointerType(E->getType()))
4491             .withConst()
4492             .withRestrict(),
4493         ImplicitParamDecl::Other));
4494     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4495     PrivateVarsPos[VD] = Counter;
4496     ++Counter;
4497   }
4498   for (const Expr *E : LastprivateVars) {
4499     Args.push_back(ImplicitParamDecl::Create(
4500         C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4501         C.getPointerType(C.getPointerType(E->getType()))
4502             .withConst()
4503             .withRestrict(),
4504         ImplicitParamDecl::Other));
4505     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4506     PrivateVarsPos[VD] = Counter;
4507     ++Counter;
4508   }
4509   const auto &TaskPrivatesMapFnInfo =
4510       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4511   llvm::FunctionType *TaskPrivatesMapTy =
4512       CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
4513   std::string Name =
4514       CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""});
4515   auto *TaskPrivatesMap = llvm::Function::Create(
4516       TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,
4517       &CGM.getModule());
4518   CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap,
4519                                     TaskPrivatesMapFnInfo);
4520   TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4521   TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4522   TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4523   CodeGenFunction CGF(CGM);
4524   CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
4525                     TaskPrivatesMapFnInfo, Args, Loc, Loc);
4526
4527   // *privi = &.privates.privi;
4528   LValue Base = CGF.EmitLoadOfPointerLValue(
4529       CGF.GetAddrOfLocalVar(&TaskPrivatesArg),
4530       TaskPrivatesArg.getType()->castAs<PointerType>());
4531   const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
4532   Counter = 0;
4533   for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
4534     LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
4535     const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4536     LValue RefLVal =
4537         CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
4538     LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
4539         RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>());
4540     CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal);
4541     ++Counter;
4542   }
4543   CGF.FinishFunction();
4544   return TaskPrivatesMap;
4545 }
4546
4547 static bool stable_sort_comparator(const PrivateDataTy P1,
4548                                    const PrivateDataTy P2) {
4549   return P1.first > P2.first;
4550 }
4551
4552 /// Emit initialization for private variables in task-based directives.
4553 static void emitPrivatesInit(CodeGenFunction &CGF,
4554                              const OMPExecutableDirective &D,
4555                              Address KmpTaskSharedsPtr, LValue TDBase,
4556                              const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4557                              QualType SharedsTy, QualType SharedsPtrTy,
4558                              const OMPTaskDataTy &Data,
4559                              ArrayRef<PrivateDataTy> Privates, bool ForDup) {
4560   ASTContext &C = CGF.getContext();
4561   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4562   LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
4563   OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
4564                                  ? OMPD_taskloop
4565                                  : OMPD_task;
4566   const CapturedStmt &CS = *D.getCapturedStmt(Kind);
4567   CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
4568   LValue SrcBase;
4569   bool IsTargetTask =
4570       isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) ||
4571       isOpenMPTargetExecutionDirective(D.getDirectiveKind());
4572   // For target-based directives skip 3 firstprivate arrays BasePointersArray,
4573   // PointersArray and SizesArray. The original variables for these arrays are
4574   // not captured and we get their addresses explicitly.
4575   if ((!IsTargetTask && !Data.FirstprivateVars.empty()) ||
4576       (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
4577     SrcBase = CGF.MakeAddrLValue(
4578         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4579             KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
4580         SharedsTy);
4581   }
4582   FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4583   for (const PrivateDataTy &Pair : Privates) {
4584     const VarDecl *VD = Pair.second.PrivateCopy;
4585     const Expr *Init = VD->getAnyInitializer();
4586     if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4587                              !CGF.isTrivialInitializer(Init)))) {
4588       LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
4589       if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
4590         const VarDecl *OriginalVD = Pair.second.Original;
4591         // Check if the variable is the target-based BasePointersArray,
4592         // PointersArray or SizesArray.
4593         LValue SharedRefLValue;
4594         QualType Type = OriginalVD->getType();
4595         const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD);
4596         if (IsTargetTask && !SharedField) {
4597           assert(isa<ImplicitParamDecl>(OriginalVD) &&
4598                  isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
4599                  cast<CapturedDecl>(OriginalVD->getDeclContext())
4600                          ->getNumParams() == 0 &&
4601                  isa<TranslationUnitDecl>(
4602                      cast<CapturedDecl>(OriginalVD->getDeclContext())
4603                          ->getDeclContext()) &&
4604                  "Expected artificial target data variable.");
4605           SharedRefLValue =
4606               CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
4607         } else {
4608           SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
4609           SharedRefLValue = CGF.MakeAddrLValue(
4610               Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
4611               SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl),
4612               SharedRefLValue.getTBAAInfo());
4613         }
4614         if (Type->isArrayType()) {
4615           // Initialize firstprivate array.
4616           if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
4617             // Perform simple memcpy.
4618             CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
4619           } else {
4620             // Initialize firstprivate array using element-by-element
4621             // initialization.
4622             CGF.EmitOMPAggregateAssign(
4623                 PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
4624                 [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
4625                                                   Address SrcElement) {
4626                   // Clean up any temporaries needed by the initialization.
4627                   CodeGenFunction::OMPPrivateScope InitScope(CGF);
4628                   InitScope.addPrivate(
4629                       Elem, [SrcElement]() -> Address { return SrcElement; });
4630                   (void)InitScope.Privatize();
4631                   // Emit initialization for single element.
4632                   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
4633                       CGF, &CapturesInfo);
4634                   CGF.EmitAnyExprToMem(Init, DestElement,
4635                                        Init->getType().getQualifiers(),
4636                                        /*IsInitializer=*/false);
4637                 });
4638           }
4639         } else {
4640           CodeGenFunction::OMPPrivateScope InitScope(CGF);
4641           InitScope.addPrivate(Elem, [SharedRefLValue]() -> Address {
4642             return SharedRefLValue.getAddress();
4643           });
4644           (void)InitScope.Privatize();
4645           CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
4646           CGF.EmitExprAsInit(Init, VD, PrivateLValue,
4647                              /*capturedByInit=*/false);
4648         }
4649       } else {
4650         CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
4651       }
4652     }
4653     ++FI;
4654   }
4655 }
4656
4657 /// Check if duplication function is required for taskloops.
4658 static bool checkInitIsRequired(CodeGenFunction &CGF,
4659                                 ArrayRef<PrivateDataTy> Privates) {
4660   bool InitRequired = false;
4661   for (const PrivateDataTy &Pair : Privates) {
4662     const VarDecl *VD = Pair.second.PrivateCopy;
4663     const Expr *Init = VD->getAnyInitializer();
4664     InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4665                                     !CGF.isTrivialInitializer(Init));
4666     if (InitRequired)
4667       break;
4668   }
4669   return InitRequired;
4670 }
4671
4672
4673 /// Emit task_dup function (for initialization of
4674 /// private/firstprivate/lastprivate vars and last_iter flag)
4675 /// \code
4676 /// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
4677 /// lastpriv) {
4678 /// // setup lastprivate flag
4679 ///    task_dst->last = lastpriv;
4680 /// // could be constructor calls here...
4681 /// }
4682 /// \endcode
4683 static llvm::Value *
4684 emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
4685                     const OMPExecutableDirective &D,
4686                     QualType KmpTaskTWithPrivatesPtrQTy,
4687                     const RecordDecl *KmpTaskTWithPrivatesQTyRD,
4688                     const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
4689                     QualType SharedsPtrTy, const OMPTaskDataTy &Data,
4690                     ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
4691   ASTContext &C = CGM.getContext();
4692   FunctionArgList Args;
4693   ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4694                            KmpTaskTWithPrivatesPtrQTy,
4695                            ImplicitParamDecl::Other);
4696   ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
4697                            KmpTaskTWithPrivatesPtrQTy,
4698                            ImplicitParamDecl::Other);
4699   ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
4700                                 ImplicitParamDecl::Other);
4701   Args.push_back(&DstArg);
4702   Args.push_back(&SrcArg);
4703   Args.push_back(&LastprivArg);
4704   const auto &TaskDupFnInfo =
4705       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
4706   llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo);
4707   std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""});
4708   auto *TaskDup = llvm::Function::Create(
4709       TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule());
4710   CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo);
4711   TaskDup->setDoesNotRecurse();
4712   CodeGenFunction CGF(CGM);
4713   CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc,
4714                     Loc);
4715
4716   LValue TDBase = CGF.EmitLoadOfPointerLValue(
4717       CGF.GetAddrOfLocalVar(&DstArg),
4718       KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4719   // task_dst->liter = lastpriv;
4720   if (WithLastIter) {
4721     auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4722     LValue Base = CGF.EmitLValueForField(
4723         TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4724     LValue LILVal = CGF.EmitLValueForField(Base, *LIFI);
4725     llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
4726         CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc);
4727     CGF.EmitStoreOfScalar(Lastpriv, LILVal);
4728   }
4729
4730   // Emit initial values for private copies (if any).
4731   assert(!Privates.empty());
4732   Address KmpTaskSharedsPtr = Address::invalid();
4733   if (!Data.FirstprivateVars.empty()) {
4734     LValue TDBase = CGF.EmitLoadOfPointerLValue(
4735         CGF.GetAddrOfLocalVar(&SrcArg),
4736         KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
4737     LValue Base = CGF.EmitLValueForField(
4738         TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
4739     KmpTaskSharedsPtr = Address(
4740         CGF.EmitLoadOfScalar(CGF.EmitLValueForField(
4741                                  Base, *std::next(KmpTaskTQTyRD->field_begin(),
4742                                                   KmpTaskTShareds)),
4743                              Loc),
4744         CGF.getNaturalTypeAlignment(SharedsTy));
4745   }
4746   emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4747                    SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
4748   CGF.FinishFunction();
4749   return TaskDup;
4750 }
4751
4752 /// Checks if destructor function is required to be generated.
4753 /// \return true if cleanups are required, false otherwise.
4754 static bool
4755 checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) {
4756   bool NeedsCleanup = false;
4757   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4758   const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4759   for (const FieldDecl *FD : PrivateRD->fields()) {
4760     NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4761     if (NeedsCleanup)
4762       break;
4763   }
4764   return NeedsCleanup;
4765 }
4766
4767 CGOpenMPRuntime::TaskResultTy
4768 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
4769                               const OMPExecutableDirective &D,
4770                               llvm::Value *TaskFunction, QualType SharedsTy,
4771                               Address Shareds, const OMPTaskDataTy &Data) {
4772   ASTContext &C = CGM.getContext();
4773   llvm::SmallVector<PrivateDataTy, 4> Privates;
4774   // Aggregate privates and sort them by the alignment.
4775   auto I = Data.PrivateCopies.begin();
4776   for (const Expr *E : Data.PrivateVars) {
4777     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4778     Privates.emplace_back(
4779         C.getDeclAlign(VD),
4780         PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4781                          /*PrivateElemInit=*/nullptr));
4782     ++I;
4783   }
4784   I = Data.FirstprivateCopies.begin();
4785   auto IElemInitRef = Data.FirstprivateInits.begin();
4786   for (const Expr *E : Data.FirstprivateVars) {
4787     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4788     Privates.emplace_back(
4789         C.getDeclAlign(VD),
4790         PrivateHelpersTy(
4791             VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4792             cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4793     ++I;
4794     ++IElemInitRef;
4795   }
4796   I = Data.LastprivateCopies.begin();
4797   for (const Expr *E : Data.LastprivateVars) {
4798     const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4799     Privates.emplace_back(
4800         C.getDeclAlign(VD),
4801         PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4802                          /*PrivateElemInit=*/nullptr));
4803     ++I;
4804   }
4805   std::stable_sort(Privates.begin(), Privates.end(), stable_sort_comparator);
4806   QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
4807   // Build type kmp_routine_entry_t (if not built yet).
4808   emitKmpRoutineEntryT(KmpInt32Ty);
4809   // Build type kmp_task_t (if not built yet).
4810   if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
4811     if (SavedKmpTaskloopTQTy.isNull()) {
4812       SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4813           CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4814     }
4815     KmpTaskTQTy = SavedKmpTaskloopTQTy;
4816   } else {
4817     assert((D.getDirectiveKind() == OMPD_task ||
4818             isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
4819             isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
4820            "Expected taskloop, task or target directive");
4821     if (SavedKmpTaskTQTy.isNull()) {
4822       SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
4823           CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
4824     }
4825     KmpTaskTQTy = SavedKmpTaskTQTy;
4826   }
4827   const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
4828   // Build particular struct kmp_task_t for the given task.
4829   const RecordDecl *KmpTaskTWithPrivatesQTyRD =
4830       createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
4831   QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
4832   QualType KmpTaskTWithPrivatesPtrQTy =
4833       C.getPointerType(KmpTaskTWithPrivatesQTy);
4834   llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
4835   llvm::Type *KmpTaskTWithPrivatesPtrTy =
4836       KmpTaskTWithPrivatesTy->getPointerTo();
4837   llvm::Value *KmpTaskTWithPrivatesTySize =
4838       CGF.getTypeSize(KmpTaskTWithPrivatesQTy);
4839   QualType SharedsPtrTy = C.getPointerType(SharedsTy);
4840
4841   // Emit initial values for private copies (if any).
4842   llvm::Value *TaskPrivatesMap = nullptr;
4843   llvm::Type *TaskPrivatesMapTy =
4844       std::next(cast<llvm::Function>(TaskFunction)->arg_begin(), 3)->getType();
4845   if (!Privates.empty()) {
4846     auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4847     TaskPrivatesMap = emitTaskPrivateMappingFunction(
4848         CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars,
4849         FI->getType(), Privates);
4850     TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4851         TaskPrivatesMap, TaskPrivatesMapTy);
4852   } else {
4853     TaskPrivatesMap = llvm::ConstantPointerNull::get(
4854         cast<llvm::PointerType>(TaskPrivatesMapTy));
4855   }
4856   // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
4857   // kmp_task_t *tt);
4858   llvm::Value *TaskEntry = emitProxyTaskFunction(
4859       CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4860       KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
4861       TaskPrivatesMap);
4862
4863   // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
4864   // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
4865   // kmp_routine_entry_t *task_entry);
4866   // Task flags. Format is taken from
4867   // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h,
4868   // description of kmp_tasking_flags struct.
4869   enum {
4870     TiedFlag = 0x1,
4871     FinalFlag = 0x2,
4872     DestructorsFlag = 0x8,
4873     PriorityFlag = 0x20
4874   };
4875   unsigned Flags = Data.Tied ? TiedFlag : 0;
4876   bool NeedsCleanup = false;
4877   if (!Privates.empty()) {
4878     NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD);
4879     if (NeedsCleanup)
4880       Flags = Flags | DestructorsFlag;
4881   }
4882   if (Data.Priority.getInt())
4883     Flags = Flags | PriorityFlag;
4884   llvm::Value *TaskFlags =
4885       Data.Final.getPointer()
4886           ? CGF.Builder.CreateSelect(Data.Final.getPointer(),
4887                                      CGF.Builder.getInt32(FinalFlag),
4888                                      CGF.Builder.getInt32(/*C=*/0))
4889           : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0);
4890   TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
4891   llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy));
4892   llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc),
4893                               getThreadID(CGF, Loc), TaskFlags,
4894                               KmpTaskTWithPrivatesTySize, SharedsSize,
4895                               CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4896                                   TaskEntry, KmpRoutineEntryPtrTy)};
4897   llvm::Value *NewTask = CGF.EmitRuntimeCall(
4898       createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
4899   llvm::Value *NewTaskNewTaskTTy =
4900       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4901           NewTask, KmpTaskTWithPrivatesPtrTy);
4902   LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
4903                                                KmpTaskTWithPrivatesQTy);
4904   LValue TDBase =
4905       CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
4906   // Fill the data in the resulting kmp_task_t record.
4907   // Copy shareds if there are any.
4908   Address KmpTaskSharedsPtr = Address::invalid();
4909   if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
4910     KmpTaskSharedsPtr =
4911         Address(CGF.EmitLoadOfScalar(
4912                     CGF.EmitLValueForField(
4913                         TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
4914                                            KmpTaskTShareds)),
4915                     Loc),
4916                 CGF.getNaturalTypeAlignment(SharedsTy));
4917     LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy);
4918     LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy);
4919     CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap);
4920   }
4921   // Emit initial values for private copies (if any).
4922   TaskResultTy Result;
4923   if (!Privates.empty()) {
4924     emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
4925                      SharedsTy, SharedsPtrTy, Data, Privates,
4926                      /*ForDup=*/false);
4927     if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
4928         (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4929       Result.TaskDupFn = emitTaskDupFunction(
4930           CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4931           KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4932           /*WithLastIter=*/!Data.LastprivateVars.empty());
4933     }
4934   }
4935   // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4936   enum { Priority = 0, Destructors = 1 };
4937   // Provide pointer to function with destructors for privates.
4938   auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
4939   const RecordDecl *KmpCmplrdataUD =
4940       (*FI)->getType()->getAsUnionType()->getDecl();
4941   if (NeedsCleanup) {
4942     llvm::Value *DestructorFn = emitDestructorsFunction(
4943         CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4944         KmpTaskTWithPrivatesQTy);
4945     LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
4946     LValue DestructorsLV = CGF.EmitLValueForField(
4947         Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
4948     CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4949                               DestructorFn, KmpRoutineEntryPtrTy),
4950                           DestructorsLV);
4951   }
4952   // Set priority.
4953   if (Data.Priority.getInt()) {
4954     LValue Data2LV = CGF.EmitLValueForField(
4955         TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
4956     LValue PriorityLV = CGF.EmitLValueForField(
4957         Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
4958     CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV);
4959   }
4960   Result.NewTask = NewTask;
4961   Result.TaskEntry = TaskEntry;
4962   Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4963   Result.TDBase = TDBase;
4964   Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4965   return Result;
4966 }
4967
4968 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4969                                    const OMPExecutableDirective &D,
4970                                    llvm::Value *TaskFunction,
4971                                    QualType SharedsTy, Address Shareds,
4972                                    const Expr *IfCond,
4973                                    const OMPTaskDataTy &Data) {
4974   if (!CGF.HaveInsertPoint())
4975     return;
4976
4977   TaskResultTy Result =
4978       emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4979   llvm::Value *NewTask = Result.NewTask;
4980   llvm::Value *TaskEntry = Result.TaskEntry;
4981   llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4982   LValue TDBase = Result.TDBase;
4983   const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4984   ASTContext &C = CGM.getContext();
4985   // Process list of dependences.
4986   Address DependenciesArray = Address::invalid();
4987   unsigned NumDependencies = Data.Dependences.size();
4988   if (NumDependencies) {
4989     // Dependence kind for RTL.
4990     enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
4991     enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
4992     RecordDecl *KmpDependInfoRD;
4993     QualType FlagsTy =
4994         C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false);
4995     llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
4996     if (KmpDependInfoTy.isNull()) {
4997       KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
4998       KmpDependInfoRD->startDefinition();
4999       addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
5000       addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
5001       addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
5002       KmpDependInfoRD->completeDefinition();
5003       KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
5004     } else {
5005       KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
5006     }
5007     CharUnits DependencySize = C.getTypeSizeInChars(KmpDependInfoTy);
5008     // Define type kmp_depend_info[<Dependences.size()>];
5009     QualType KmpDependInfoArrayTy = C.getConstantArrayType(
5010         KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies),
5011         ArrayType::Normal, /*IndexTypeQuals=*/0);
5012     // kmp_depend_info[<Dependences.size()>] deps;
5013     DependenciesArray =
5014         CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr");
5015     for (unsigned I = 0; I < NumDependencies; ++I) {
5016       const Expr *E = Data.Dependences[I].second;
5017       LValue Addr = CGF.EmitLValue(E);
5018       llvm::Value *Size;
5019       QualType Ty = E->getType();
5020       if (const auto *ASE =
5021               dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) {
5022         LValue UpAddrLVal =
5023             CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false);
5024         llvm::Value *UpAddr =
5025             CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1);
5026         llvm::Value *LowIntPtr =
5027             CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
5028         llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy);
5029         Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5030       } else {
5031         Size = CGF.getTypeSize(Ty);
5032       }
5033       LValue Base = CGF.MakeAddrLValue(
5034           CGF.Builder.CreateConstArrayGEP(DependenciesArray, I, DependencySize),
5035           KmpDependInfoTy);
5036       // deps[i].base_addr = &<Dependences[i].second>;
5037       LValue BaseAddrLVal = CGF.EmitLValueForField(
5038           Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
5039       CGF.EmitStoreOfScalar(
5040           CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
5041           BaseAddrLVal);
5042       // deps[i].len = sizeof(<Dependences[i].second>);
5043       LValue LenLVal = CGF.EmitLValueForField(
5044           Base, *std::next(KmpDependInfoRD->field_begin(), Len));
5045       CGF.EmitStoreOfScalar(Size, LenLVal);
5046       // deps[i].flags = <Dependences[i].first>;
5047       RTLDependenceKindTy DepKind;
5048       switch (Data.Dependences[I].first) {
5049       case OMPC_DEPEND_in:
5050         DepKind = DepIn;
5051         break;
5052       // Out and InOut dependencies must use the same code.
5053       case OMPC_DEPEND_out:
5054       case OMPC_DEPEND_inout:
5055         DepKind = DepInOut;
5056         break;
5057       case OMPC_DEPEND_source:
5058       case OMPC_DEPEND_sink:
5059       case OMPC_DEPEND_unknown:
5060         llvm_unreachable("Unknown task dependence type");
5061       }
5062       LValue FlagsLVal = CGF.EmitLValueForField(
5063           Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
5064       CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
5065                             FlagsLVal);
5066     }
5067     DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5068         CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()),
5069         CGF.VoidPtrTy);
5070   }
5071
5072   // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5073   // libcall.
5074   // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
5075   // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
5076   // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
5077   // list is not empty
5078   llvm::Value *ThreadID = getThreadID(CGF, Loc);
5079   llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5080   llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5081   llvm::Value *DepTaskArgs[7];
5082   if (NumDependencies) {
5083     DepTaskArgs[0] = UpLoc;
5084     DepTaskArgs[1] = ThreadID;
5085     DepTaskArgs[2] = NewTask;
5086     DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
5087     DepTaskArgs[4] = DependenciesArray.getPointer();
5088     DepTaskArgs[5] = CGF.Builder.getInt32(0);
5089     DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5090   }
5091   auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5092                         &TaskArgs,
5093                         &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
5094     if (!Data.Tied) {
5095       auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5096       LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5097       CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5098     }
5099     if (NumDependencies) {
5100       CGF.EmitRuntimeCall(
5101           createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs);
5102     } else {
5103       CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task),
5104                           TaskArgs);
5105     }
5106     // Check if parent region is untied and build return for untied task;
5107     if (auto *Region =
5108             dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5109       Region->emitUntiedSwitch(CGF);
5110   };
5111
5112   llvm::Value *DepWaitTaskArgs[6];
5113   if (NumDependencies) {
5114     DepWaitTaskArgs[0] = UpLoc;
5115     DepWaitTaskArgs[1] = ThreadID;
5116     DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5117     DepWaitTaskArgs[3] = DependenciesArray.getPointer();
5118     DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5119     DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5120   }
5121   auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5122                         NumDependencies, &DepWaitTaskArgs,
5123                         Loc](CodeGenFunction &CGF, PrePostActionTy &) {
5124     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5125     CodeGenFunction::RunCleanupsScope LocalScope(CGF);
5126     // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
5127     // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
5128     // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
5129     // is specified.
5130     if (NumDependencies)
5131       CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
5132                           DepWaitTaskArgs);
5133     // Call proxy_task_entry(gtid, new_task);
5134     auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5135                       Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
5136       Action.Enter(CGF);
5137       llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5138       CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5139                                                           OutlinedFnArgs);
5140     };
5141
5142     // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
5143     // kmp_task_t *new_task);
5144     // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
5145     // kmp_task_t *new_task);
5146     RegionCodeGenTy RCG(CodeGen);
5147     CommonActionTy Action(
5148         RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs,
5149         RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs);
5150     RCG.setAction(Action);
5151     RCG(CGF);
5152   };
5153
5154   if (IfCond) {
5155     emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
5156   } else {
5157     RegionCodeGenTy ThenRCG(ThenCodeGen);
5158     ThenRCG(CGF);
5159   }
5160 }
5161
5162 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
5163                                        const OMPLoopDirective &D,
5164                                        llvm::Value *TaskFunction,
5165                                        QualType SharedsTy, Address Shareds,
5166                                        const Expr *IfCond,
5167                                        const OMPTaskDataTy &Data) {
5168   if (!CGF.HaveInsertPoint())
5169     return;
5170   TaskResultTy Result =
5171       emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5172   // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
5173   // libcall.
5174   // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
5175   // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
5176   // sched, kmp_uint64 grainsize, void *task_dup);
5177   llvm::Value *ThreadID = getThreadID(CGF, Loc);
5178   llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
5179   llvm::Value *IfVal;
5180   if (IfCond) {
5181     IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
5182                                       /*isSigned=*/true);
5183   } else {
5184     IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
5185   }
5186
5187   LValue LBLVal = CGF.EmitLValueForField(
5188       Result.TDBase,
5189       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
5190   const auto *LBVar =
5191       cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
5192   CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
5193                        /*IsInitializer=*/true);
5194   LValue UBLVal = CGF.EmitLValueForField(
5195       Result.TDBase,
5196       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
5197   const auto *UBVar =
5198       cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
5199   CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
5200                        /*IsInitializer=*/true);
5201   LValue StLVal = CGF.EmitLValueForField(
5202       Result.TDBase,
5203       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
5204   const auto *StVar =
5205       cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
5206   CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
5207                        /*IsInitializer=*/true);
5208   // Store reductions address.
5209   LValue RedLVal = CGF.EmitLValueForField(
5210       Result.TDBase,
5211       *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
5212   if (Data.Reductions) {
5213     CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
5214   } else {
5215     CGF.EmitNullInitialization(RedLVal.getAddress(),
5216                                CGF.getContext().VoidPtrTy);
5217   }
5218   enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5219   llvm::Value *TaskArgs[] = {
5220       UpLoc,
5221       ThreadID,
5222       Result.NewTask,
5223       IfVal,
5224       LBLVal.getPointer(),
5225       UBLVal.getPointer(),
5226       CGF.EmitLoadOfScalar(StLVal, Loc),
5227       llvm::ConstantInt::getNullValue(
5228           CGF.IntTy), // Always 0 because taskgroup emitted by the compiler
5229       llvm::ConstantInt::getSigned(
5230           CGF.IntTy, Data.Schedule.getPointer()
5231                          ? Data.Schedule.getInt() ? NumTasks : Grainsize
5232                          : NoSchedule),
5233       Data.Schedule.getPointer()
5234           ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
5235                                       /*isSigned=*/false)
5236           : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0),
5237       Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5238                              Result.TaskDupFn, CGF.VoidPtrTy)
5239                        : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)};
5240   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs);
5241 }
5242
5243 /// Emit reduction operation for each element of array (required for
5244 /// array sections) LHS op = RHS.
5245 /// \param Type Type of array.
5246 /// \param LHSVar Variable on the left side of the reduction operation
5247 /// (references element of array in original variable).
5248 /// \param RHSVar Variable on the right side of the reduction operation
5249 /// (references element of array in original variable).
5250 /// \param RedOpGen Generator of reduction operation with use of LHSVar and
5251 /// RHSVar.
5252 static void EmitOMPAggregateReduction(
5253     CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
5254     const VarDecl *RHSVar,
5255     const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
5256                                   const Expr *, const Expr *)> &RedOpGen,
5257     const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
5258     const Expr *UpExpr = nullptr) {
5259   // Perform element-by-element initialization.
5260   QualType ElementTy;
5261   Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar);
5262   Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar);
5263
5264   // Drill down to the base element type on both arrays.
5265   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
5266   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
5267
5268   llvm::Value *RHSBegin = RHSAddr.getPointer();
5269   llvm::Value *LHSBegin = LHSAddr.getPointer();
5270   // Cast from pointer to array type to pointer to single element.
5271   llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements);
5272   // The basic structure here is a while-do loop.
5273   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body");
5274   llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done");
5275   llvm::Value *IsEmpty =
5276       CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty");
5277   CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5278
5279   // Enter the loop body, making that address the current address.
5280   llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
5281   CGF.EmitBlock(BodyBB);
5282
5283   CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
5284
5285   llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
5286       RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast");
5287   RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5288   Address RHSElementCurrent =
5289       Address(RHSElementPHI,
5290               RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5291
5292   llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
5293       LHSBegin->getType(), 2, "omp.arraycpy.destElementPast");
5294   LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5295   Address LHSElementCurrent =
5296       Address(LHSElementPHI,
5297               LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize));
5298
5299   // Emit copy.
5300   CodeGenFunction::OMPPrivateScope Scope(CGF);
5301   Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; });
5302   Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; });
5303   Scope.Privatize();
5304   RedOpGen(CGF, XExpr, EExpr, UpExpr);
5305   Scope.ForceCleanup();
5306
5307   // Shift the address forward by one element.
5308   llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
5309       LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
5310   llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
5311       RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
5312   // Check whether we've reached the end.
5313   llvm::Value *Done =
5314       CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done");
5315   CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
5316   LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock());
5317   RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock());
5318
5319   // Done.
5320   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
5321 }
5322
5323 /// Emit reduction combiner. If the combiner is a simple expression emit it as
5324 /// is, otherwise consider it as combiner of UDR decl and emit it as a call of
5325 /// UDR combiner function.
5326 static void emitReductionCombiner(CodeGenFunction &CGF,
5327                                   const Expr *ReductionOp) {
5328   if (const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5329     if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5330       if (const auto *DRE =
5331               dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5332         if (const auto *DRD =
5333                 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5334           std::pair<llvm::Function *, llvm::Function *> Reduction =
5335               CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
5336           RValue Func = RValue::get(Reduction.first);
5337           CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
5338           CGF.EmitIgnoredExpr(ReductionOp);
5339           return;
5340         }
5341   CGF.EmitIgnoredExpr(ReductionOp);
5342 }
5343
5344 llvm::Value *CGOpenMPRuntime::emitReductionFunction(
5345     CodeGenModule &CGM, SourceLocation Loc, llvm::Type *ArgsType,
5346     ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
5347     ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
5348   ASTContext &C = CGM.getContext();
5349
5350   // void reduction_func(void *LHSArg, void *RHSArg);
5351   FunctionArgList Args;
5352   ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5353                            ImplicitParamDecl::Other);
5354   ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5355                            ImplicitParamDecl::Other);
5356   Args.push_back(&LHSArg);
5357   Args.push_back(&RHSArg);
5358   const auto &CGFI =
5359       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5360   std::string Name = getName({"omp", "reduction", "reduction_func"});
5361   auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
5362                                     llvm::GlobalValue::InternalLinkage, Name,
5363                                     &CGM.getModule());
5364   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
5365   Fn->setDoesNotRecurse();
5366   CodeGenFunction CGF(CGM);
5367   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
5368
5369   // Dst = (void*[n])(LHSArg);
5370   // Src = (void*[n])(RHSArg);
5371   Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5372       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)),
5373       ArgsType), CGF.getPointerAlign());
5374   Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5375       CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)),
5376       ArgsType), CGF.getPointerAlign());
5377
5378   //  ...
5379   //  *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5380   //  ...
5381   CodeGenFunction::OMPPrivateScope Scope(CGF);
5382   auto IPriv = Privates.begin();
5383   unsigned Idx = 0;
5384   for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5385     const auto *RHSVar =
5386         cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5387     Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5388       return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar);
5389     });
5390     const auto *LHSVar =
5391         cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5392     Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5393       return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar);
5394     });
5395     QualType PrivTy = (*IPriv)->getType();
5396     if (PrivTy->isVariablyModifiedType()) {
5397       // Get array size and emit VLA type.
5398       ++Idx;
5399       Address Elem =
5400           CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize());
5401       llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem);
5402       const VariableArrayType *VLA =
5403           CGF.getContext().getAsVariableArrayType(PrivTy);
5404       const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
5405       CodeGenFunction::OpaqueValueMapping OpaqueMap(
5406           CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy)));
5407       CGF.EmitVariablyModifiedType(PrivTy);
5408     }
5409   }
5410   Scope.Privatize();
5411   IPriv = Privates.begin();
5412   auto ILHS = LHSExprs.begin();
5413   auto IRHS = RHSExprs.begin();
5414   for (const Expr *E : ReductionOps) {
5415     if ((*IPriv)->getType()->isArrayType()) {
5416       // Emit reduction for array section.
5417       const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5418       const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5419       EmitOMPAggregateReduction(
5420           CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5421           [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5422             emitReductionCombiner(CGF, E);
5423           });
5424     } else {
5425       // Emit reduction for array subscript or single variable.
5426       emitReductionCombiner(CGF, E);
5427     }
5428     ++IPriv;
5429     ++ILHS;
5430     ++IRHS;
5431   }
5432   Scope.ForceCleanup();
5433   CGF.FinishFunction();
5434   return Fn;
5435 }
5436
5437 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5438                                                   const Expr *ReductionOp,
5439                                                   const Expr *PrivateRef,
5440                                                   const DeclRefExpr *LHS,
5441                                                   const DeclRefExpr *RHS) {
5442   if (PrivateRef->getType()->isArrayType()) {
5443     // Emit reduction for array section.
5444     const auto *LHSVar = cast<VarDecl>(LHS->getDecl());
5445     const auto *RHSVar = cast<VarDecl>(RHS->getDecl());
5446     EmitOMPAggregateReduction(
5447         CGF, PrivateRef->getType(), LHSVar, RHSVar,
5448         [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5449           emitReductionCombiner(CGF, ReductionOp);
5450         });
5451   } else {
5452     // Emit reduction for array subscript or single variable.
5453     emitReductionCombiner(CGF, ReductionOp);
5454   }
5455 }
5456
5457 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5458                                     ArrayRef<const Expr *> Privates,
5459                                     ArrayRef<const Expr *> LHSExprs,
5460                                     ArrayRef<const Expr *> RHSExprs,
5461                                     ArrayRef<const Expr *> ReductionOps,
5462                                     ReductionOptionsTy Options) {
5463   if (!CGF.HaveInsertPoint())
5464     return;
5465
5466   bool WithNowait = Options.WithNowait;
5467   bool SimpleReduction = Options.SimpleReduction;
5468
5469   // Next code should be emitted for reduction:
5470   //
5471   // static kmp_critical_name lock = { 0 };
5472   //
5473   // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5474   //  *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5475   //  ...
5476   //  *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5477   //  *(Type<n>-1*)rhs[<n>-1]);
5478   // }
5479   //
5480   // ...
5481   // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5482   // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5483   // RedList, reduce_func, &<lock>)) {
5484   // case 1:
5485   //  ...
5486   //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5487   //  ...
5488   // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5489   // break;
5490   // case 2:
5491   //  ...
5492   //  Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5493   //  ...
5494   // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5495   // break;
5496   // default:;
5497   // }
5498   //
5499   // if SimpleReduction is true, only the next code is generated:
5500   //  ...
5501   //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5502   //  ...
5503
5504   ASTContext &C = CGM.getContext();
5505
5506   if (SimpleReduction) {
5507     CodeGenFunction::RunCleanupsScope Scope(CGF);
5508     auto IPriv = Privates.begin();
5509     auto ILHS = LHSExprs.begin();
5510     auto IRHS = RHSExprs.begin();
5511     for (const Expr *E : ReductionOps) {
5512       emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5513                                   cast<DeclRefExpr>(*IRHS));
5514       ++IPriv;
5515       ++ILHS;
5516       ++IRHS;
5517     }
5518     return;
5519   }
5520
5521   // 1. Build a list of reduction variables.
5522   // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5523   auto Size = RHSExprs.size();
5524   for (const Expr *E : Privates) {
5525     if (E->getType()->isVariablyModifiedType())
5526       // Reserve place for array size.
5527       ++Size;
5528   }
5529   llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5530   QualType ReductionArrayTy =
5531       C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
5532                              /*IndexTypeQuals=*/0);
5533   Address ReductionList =
5534       CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
5535   auto IPriv = Privates.begin();
5536   unsigned Idx = 0;
5537   for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5538     Address Elem =
5539       CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize());
5540     CGF.Builder.CreateStore(
5541         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5542             CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy),
5543         Elem);
5544     if ((*IPriv)->getType()->isVariablyModifiedType()) {
5545       // Store array size.
5546       ++Idx;
5547       Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx,
5548                                              CGF.getPointerSize());
5549       llvm::Value *Size = CGF.Builder.CreateIntCast(
5550           CGF.getVLASize(
5551                  CGF.getContext().getAsVariableArrayType((*IPriv)->getType()))
5552               .NumElts,
5553           CGF.SizeTy, /*isSigned=*/false);
5554       CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy),
5555                               Elem);
5556     }
5557   }
5558
5559   // 2. Emit reduce_func().
5560   llvm::Value *ReductionFn = emitReductionFunction(
5561       CGM, Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(),
5562       Privates, LHSExprs, RHSExprs, ReductionOps);
5563
5564   // 3. Create static kmp_critical_name lock = { 0 };
5565   std::string Name = getName({"reduction"});
5566   llvm::Value *Lock = getCriticalRegionLock(Name);
5567
5568   // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5569   // RedList, reduce_func, &<lock>);
5570   llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE);
5571   llvm::Value *ThreadId = getThreadID(CGF, Loc);
5572   llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy);
5573   llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5574       ReductionList.getPointer(), CGF.VoidPtrTy);
5575   llvm::Value *Args[] = {
5576       IdentTLoc,                             // ident_t *<loc>
5577       ThreadId,                              // i32 <gtid>
5578       CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
5579       ReductionArrayTySize,                  // size_type sizeof(RedList)
5580       RL,                                    // void *RedList
5581       ReductionFn, // void (*) (void *, void *) <reduce_func>
5582       Lock         // kmp_critical_name *&<lock>
5583   };
5584   llvm::Value *Res = CGF.EmitRuntimeCall(
5585       createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
5586                                        : OMPRTL__kmpc_reduce),
5587       Args);
5588
5589   // 5. Build switch(res)
5590   llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
5591   llvm::SwitchInst *SwInst =
5592       CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
5593
5594   // 6. Build case 1:
5595   //  ...
5596   //  <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5597   //  ...
5598   // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5599   // break;
5600   llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
5601   SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
5602   CGF.EmitBlock(Case1BB);
5603
5604   // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5605   llvm::Value *EndArgs[] = {
5606       IdentTLoc, // ident_t *<loc>
5607       ThreadId,  // i32 <gtid>
5608       Lock       // kmp_critical_name *&<lock>
5609   };
5610   auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5611                        CodeGenFunction &CGF, PrePostActionTy &Action) {
5612     CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5613     auto IPriv = Privates.begin();
5614     auto ILHS = LHSExprs.begin();
5615     auto IRHS = RHSExprs.begin();
5616     for (const Expr *E : ReductionOps) {
5617       RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5618                                      cast<DeclRefExpr>(*IRHS));
5619       ++IPriv;
5620       ++ILHS;
5621       ++IRHS;
5622     }
5623   };
5624   RegionCodeGenTy RCG(CodeGen);
5625   CommonActionTy Action(
5626       nullptr, llvm::None,
5627       createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
5628                                        : OMPRTL__kmpc_end_reduce),
5629       EndArgs);
5630   RCG.setAction(Action);
5631   RCG(CGF);
5632
5633   CGF.EmitBranch(DefaultBB);
5634
5635   // 7. Build case 2:
5636   //  ...
5637   //  Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5638   //  ...
5639   // break;
5640   llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
5641   SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5642   CGF.EmitBlock(Case2BB);
5643
5644   auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5645                              CodeGenFunction &CGF, PrePostActionTy &Action) {
5646     auto ILHS = LHSExprs.begin();
5647     auto IRHS = RHSExprs.begin();
5648     auto IPriv = Privates.begin();
5649     for (const Expr *E : ReductionOps) {
5650       const Expr *XExpr = nullptr;
5651       const Expr *EExpr = nullptr;
5652       const Expr *UpExpr = nullptr;
5653       BinaryOperatorKind BO = BO_Comma;
5654       if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
5655         if (BO->getOpcode() == BO_Assign) {
5656           XExpr = BO->getLHS();
5657           UpExpr = BO->getRHS();
5658         }
5659       }
5660       // Try to emit update expression as a simple atomic.
5661       const Expr *RHSExpr = UpExpr;
5662       if (RHSExpr) {
5663         // Analyze RHS part of the whole expression.
5664         if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5665                 RHSExpr->IgnoreParenImpCasts())) {
5666           // If this is a conditional operator, analyze its condition for
5667           // min/max reduction operator.
5668           RHSExpr = ACO->getCond();
5669         }
5670         if (const auto *BORHS =
5671                 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
5672           EExpr = BORHS->getRHS();
5673           BO = BORHS->getOpcode();
5674         }
5675       }
5676       if (XExpr) {
5677         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5678         auto &&AtomicRedGen = [BO, VD,
5679                                Loc](CodeGenFunction &CGF, const Expr *XExpr,
5680                                     const Expr *EExpr, const Expr *UpExpr) {
5681           LValue X = CGF.EmitLValue(XExpr);
5682           RValue E;
5683           if (EExpr)
5684             E = CGF.EmitAnyExpr(EExpr);
5685           CGF.EmitOMPAtomicSimpleUpdateExpr(
5686               X, E, BO, /*IsXLHSInRHSPart=*/true,
5687               llvm::AtomicOrdering::Monotonic, Loc,
5688               [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5689                 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5690                 PrivateScope.addPrivate(
5691                     VD, [&CGF, VD, XRValue, Loc]() {
5692                       Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5693                       CGF.emitOMPSimpleStore(
5694                           CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue,
5695                           VD->getType().getNonReferenceType(), Loc);
5696                       return LHSTemp;
5697                     });
5698                 (void)PrivateScope.Privatize();
5699                 return CGF.EmitAnyExpr(UpExpr);
5700               });
5701         };
5702         if ((*IPriv)->getType()->isArrayType()) {
5703           // Emit atomic reduction for array section.
5704           const auto *RHSVar =
5705               cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5706           EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar,
5707                                     AtomicRedGen, XExpr, EExpr, UpExpr);
5708         } else {
5709           // Emit atomic reduction for array subscript or single variable.
5710           AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5711         }
5712       } else {
5713         // Emit as a critical region.
5714         auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5715                                            const Expr *, const Expr *) {
5716           CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5717           std::string Name = RT.getName({"atomic_reduction"});
5718           RT.emitCriticalRegion(
5719               CGF, Name,
5720               [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5721                 Action.Enter(CGF);
5722                 emitReductionCombiner(CGF, E);
5723               },
5724               Loc);
5725         };
5726         if ((*IPriv)->getType()->isArrayType()) {
5727           const auto *LHSVar =
5728               cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5729           const auto *RHSVar =
5730               cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5731           EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5732                                     CritRedGen);
5733         } else {
5734           CritRedGen(CGF, nullptr, nullptr, nullptr);
5735         }
5736       }
5737       ++ILHS;
5738       ++IRHS;
5739       ++IPriv;
5740     }
5741   };
5742   RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5743   if (!WithNowait) {
5744     // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5745     llvm::Value *EndArgs[] = {
5746         IdentTLoc, // ident_t *<loc>
5747         ThreadId,  // i32 <gtid>
5748         Lock       // kmp_critical_name *&<lock>
5749     };
5750     CommonActionTy Action(nullptr, llvm::None,
5751                           createRuntimeFunction(OMPRTL__kmpc_end_reduce),
5752                           EndArgs);
5753     AtomicRCG.setAction(Action);
5754     AtomicRCG(CGF);
5755   } else {
5756     AtomicRCG(CGF);
5757   }
5758
5759   CGF.EmitBranch(DefaultBB);
5760   CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
5761 }
5762
5763 /// Generates unique name for artificial threadprivate variables.
5764 /// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5765 static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5766                                       const Expr *Ref) {
5767   SmallString<256> Buffer;
5768   llvm::raw_svector_ostream Out(Buffer);
5769   const clang::DeclRefExpr *DE;
5770   const VarDecl *D = ::getBaseDecl(Ref, DE);
5771   if (!D)
5772     D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5773   D = D->getCanonicalDecl();
5774   std::string Name = CGM.getOpenMPRuntime().getName(
5775       {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)});
5776   Out << Prefix << Name << "_"
5777       << D->getCanonicalDecl()->getLocStart().getRawEncoding();
5778   return Out.str();
5779 }
5780
5781 /// Emits reduction initializer function:
5782 /// \code
5783 /// void @.red_init(void* %arg) {
5784 /// %0 = bitcast void* %arg to <type>*
5785 /// store <type> <init>, <type>* %0
5786 /// ret void
5787 /// }
5788 /// \endcode
5789 static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5790                                            SourceLocation Loc,
5791                                            ReductionCodeGen &RCG, unsigned N) {
5792   ASTContext &C = CGM.getContext();
5793   FunctionArgList Args;
5794   ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5795                           ImplicitParamDecl::Other);
5796   Args.emplace_back(&Param);
5797   const auto &FnInfo =
5798       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5799   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5800   std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""});
5801   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5802                                     Name, &CGM.getModule());
5803   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
5804   Fn->setDoesNotRecurse();
5805   CodeGenFunction CGF(CGM);
5806   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5807   Address PrivateAddr = CGF.EmitLoadOfPointer(
5808       CGF.GetAddrOfLocalVar(&Param),
5809       C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5810   llvm::Value *Size = nullptr;
5811   // If the size of the reduction item is non-constant, load it from global
5812   // threadprivate variable.
5813   if (RCG.getSizes(N).second) {
5814     Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5815         CGF, CGM.getContext().getSizeType(),
5816         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
5817     Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5818                                 CGM.getContext().getSizeType(), Loc);
5819   }
5820   RCG.emitAggregateType(CGF, N, Size);
5821   LValue SharedLVal;
5822   // If initializer uses initializer from declare reduction construct, emit a
5823   // pointer to the address of the original reduction item (reuired by reduction
5824   // initializer)
5825   if (RCG.usesReductionInitializer(N)) {
5826     Address SharedAddr =
5827         CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5828             CGF, CGM.getContext().VoidPtrTy,
5829             generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
5830     SharedAddr = CGF.EmitLoadOfPointer(
5831         SharedAddr,
5832         CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5833     SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy);
5834   } else {
5835     SharedLVal = CGF.MakeNaturalAlignAddrLValue(
5836         llvm::ConstantPointerNull::get(CGM.VoidPtrTy),
5837         CGM.getContext().VoidPtrTy);
5838   }
5839   // Emit the initializer:
5840   // %0 = bitcast void* %arg to <type>*
5841   // store <type> <init>, <type>* %0
5842   RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal,
5843                          [](CodeGenFunction &) { return false; });
5844   CGF.FinishFunction();
5845   return Fn;
5846 }
5847
5848 /// Emits reduction combiner function:
5849 /// \code
5850 /// void @.red_comb(void* %arg0, void* %arg1) {
5851 /// %lhs = bitcast void* %arg0 to <type>*
5852 /// %rhs = bitcast void* %arg1 to <type>*
5853 /// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5854 /// store <type> %2, <type>* %lhs
5855 /// ret void
5856 /// }
5857 /// \endcode
5858 static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5859                                            SourceLocation Loc,
5860                                            ReductionCodeGen &RCG, unsigned N,
5861                                            const Expr *ReductionOp,
5862                                            const Expr *LHS, const Expr *RHS,
5863                                            const Expr *PrivateRef) {
5864   ASTContext &C = CGM.getContext();
5865   const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
5866   const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
5867   FunctionArgList Args;
5868   ImplicitParamDecl ParamInOut(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
5869                                C.VoidPtrTy, ImplicitParamDecl::Other);
5870   ImplicitParamDecl ParamIn(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5871                             ImplicitParamDecl::Other);
5872   Args.emplace_back(&ParamInOut);
5873   Args.emplace_back(&ParamIn);
5874   const auto &FnInfo =
5875       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5876   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5877   std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""});
5878   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5879                                     Name, &CGM.getModule());
5880   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
5881   Fn->setDoesNotRecurse();
5882   CodeGenFunction CGF(CGM);
5883   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5884   llvm::Value *Size = nullptr;
5885   // If the size of the reduction item is non-constant, load it from global
5886   // threadprivate variable.
5887   if (RCG.getSizes(N).second) {
5888     Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5889         CGF, CGM.getContext().getSizeType(),
5890         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
5891     Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5892                                 CGM.getContext().getSizeType(), Loc);
5893   }
5894   RCG.emitAggregateType(CGF, N, Size);
5895   // Remap lhs and rhs variables to the addresses of the function arguments.
5896   // %lhs = bitcast void* %arg0 to <type>*
5897   // %rhs = bitcast void* %arg1 to <type>*
5898   CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5899   PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
5900     // Pull out the pointer to the variable.
5901     Address PtrAddr = CGF.EmitLoadOfPointer(
5902         CGF.GetAddrOfLocalVar(&ParamInOut),
5903         C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5904     return CGF.Builder.CreateElementBitCast(
5905         PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType()));
5906   });
5907   PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
5908     // Pull out the pointer to the variable.
5909     Address PtrAddr = CGF.EmitLoadOfPointer(
5910         CGF.GetAddrOfLocalVar(&ParamIn),
5911         C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5912     return CGF.Builder.CreateElementBitCast(
5913         PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType()));
5914   });
5915   PrivateScope.Privatize();
5916   // Emit the combiner body:
5917   // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
5918   // store <type> %2, <type>* %lhs
5919   CGM.getOpenMPRuntime().emitSingleReductionCombiner(
5920       CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
5921       cast<DeclRefExpr>(RHS));
5922   CGF.FinishFunction();
5923   return Fn;
5924 }
5925
5926 /// Emits reduction finalizer function:
5927 /// \code
5928 /// void @.red_fini(void* %arg) {
5929 /// %0 = bitcast void* %arg to <type>*
5930 /// <destroy>(<type>* %0)
5931 /// ret void
5932 /// }
5933 /// \endcode
5934 static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
5935                                            SourceLocation Loc,
5936                                            ReductionCodeGen &RCG, unsigned N) {
5937   if (!RCG.needCleanups(N))
5938     return nullptr;
5939   ASTContext &C = CGM.getContext();
5940   FunctionArgList Args;
5941   ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy,
5942                           ImplicitParamDecl::Other);
5943   Args.emplace_back(&Param);
5944   const auto &FnInfo =
5945       CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5946   llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
5947   std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""});
5948   auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
5949                                     Name, &CGM.getModule());
5950   CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
5951   Fn->setDoesNotRecurse();
5952   CodeGenFunction CGF(CGM);
5953   CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
5954   Address PrivateAddr = CGF.EmitLoadOfPointer(
5955       CGF.GetAddrOfLocalVar(&Param),
5956       C.getPointerType(C.VoidPtrTy).castAs<PointerType>());
5957   llvm::Value *Size = nullptr;
5958   // If the size of the reduction item is non-constant, load it from global
5959   // threadprivate variable.
5960   if (RCG.getSizes(N).second) {
5961     Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5962         CGF, CGM.getContext().getSizeType(),
5963         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
5964     Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false,
5965                                 CGM.getContext().getSizeType(), Loc);
5966   }
5967   RCG.emitAggregateType(CGF, N, Size);
5968   // Emit the finalizer body:
5969   // <destroy>(<type>* %0)
5970   RCG.emitCleanups(CGF, N, PrivateAddr);
5971   CGF.FinishFunction();
5972   return Fn;
5973 }
5974
5975 llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
5976     CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5977     ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
5978   if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5979     return nullptr;
5980
5981   // Build typedef struct:
5982   // kmp_task_red_input {
5983   //   void *reduce_shar; // shared reduction item
5984   //   size_t reduce_size; // size of data item
5985   //   void *reduce_init; // data initialization routine
5986   //   void *reduce_fini; // data finalization routine
5987   //   void *reduce_comb; // data combiner routine
5988   //   kmp_task_red_flags_t flags; // flags for additional info from compiler
5989   // } kmp_task_red_input_t;
5990   ASTContext &C = CGM.getContext();
5991   RecordDecl *RD = C.buildImplicitRecord("kmp_task_red_input_t");
5992   RD->startDefinition();
5993   const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5994   const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType());
5995   const FieldDecl *InitFD  = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5996   const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5997   const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy);
5998   const FieldDecl *FlagsFD = addFieldToRecordDecl(
5999       C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
6000   RD->completeDefinition();
6001   QualType RDType = C.getRecordType(RD);
6002   unsigned Size = Data.ReductionVars.size();
6003   llvm::APInt ArraySize(/*numBits=*/64, Size);
6004   QualType ArrayRDType = C.getConstantArrayType(
6005       RDType, ArraySize, ArrayType::Normal, /*IndexTypeQuals=*/0);
6006   // kmp_task_red_input_t .rd_input.[Size];
6007   Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
6008   ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionCopies,
6009                        Data.ReductionOps);
6010   for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6011     // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
6012     llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0),
6013                            llvm::ConstantInt::get(CGM.SizeTy, Cnt)};
6014     llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
6015         TaskRedInput.getPointer(), Idxs,
6016         /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
6017         ".rd_input.gep.");
6018     LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
6019     // ElemLVal.reduce_shar = &Shareds[Cnt];
6020     LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
6021     RCG.emitSharedLValue(CGF, Cnt);
6022     llvm::Value *CastedShared =
6023         CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer());
6024     CGF.EmitStoreOfScalar(CastedShared, SharedLVal);
6025     RCG.emitAggregateType(CGF, Cnt);
6026     llvm::Value *SizeValInChars;
6027     llvm::Value *SizeVal;
6028     std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt);
6029     // We use delayed creation/initialization for VLAs, array sections and
6030     // custom reduction initializations. It is required because runtime does not
6031     // provide the way to pass the sizes of VLAs/array sections to
6032     // initializer/combiner/finalizer functions and does not pass the pointer to
6033     // original reduction item to the initializer. Instead threadprivate global
6034     // variables are used to store these values and use them in the functions.
6035     bool DelayedCreation = !!SizeVal;
6036     SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy,
6037                                                /*isSigned=*/false);
6038     LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD);
6039     CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal);
6040     // ElemLVal.reduce_init = init;
6041     LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD);
6042     llvm::Value *InitAddr =
6043         CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt));
6044     CGF.EmitStoreOfScalar(InitAddr, InitLVal);
6045     DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt);
6046     // ElemLVal.reduce_fini = fini;
6047     LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD);
6048     llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt);
6049     llvm::Value *FiniAddr = Fini
6050                                 ? CGF.EmitCastToVoidPtr(Fini)
6051                                 : llvm::ConstantPointerNull::get(CGM.VoidPtrTy);
6052     CGF.EmitStoreOfScalar(FiniAddr, FiniLVal);
6053     // ElemLVal.reduce_comb = comb;
6054     LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD);
6055     llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction(
6056         CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt],
6057         RHSExprs[Cnt], Data.ReductionCopies[Cnt]));
6058     CGF.EmitStoreOfScalar(CombAddr, CombLVal);
6059     // ElemLVal.flags = 0;
6060     LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD);
6061     if (DelayedCreation) {
6062       CGF.EmitStoreOfScalar(
6063           llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*IsSigned=*/true),
6064           FlagsLVal);
6065     } else
6066       CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType());
6067   }
6068   // Build call void *__kmpc_task_reduction_init(int gtid, int num_data, void
6069   // *data);
6070   llvm::Value *Args[] = {
6071       CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6072                                 /*isSigned=*/true),
6073       llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
6074       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
6075                                                       CGM.VoidPtrTy)};
6076   return CGF.EmitRuntimeCall(
6077       createRuntimeFunction(OMPRTL__kmpc_task_reduction_init), Args);
6078 }
6079
6080 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6081                                               SourceLocation Loc,
6082                                               ReductionCodeGen &RCG,
6083                                               unsigned N) {
6084   auto Sizes = RCG.getSizes(N);
6085   // Emit threadprivate global variable if the type is non-constant
6086   // (Sizes.second = nullptr).
6087   if (Sizes.second) {
6088     llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy,
6089                                                      /*isSigned=*/false);
6090     Address SizeAddr = getAddrOfArtificialThreadPrivate(
6091         CGF, CGM.getContext().getSizeType(),
6092         generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N)));
6093     CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false);
6094   }
6095   // Store address of the original reduction item if custom initializer is used.
6096   if (RCG.usesReductionInitializer(N)) {
6097     Address SharedAddr = getAddrOfArtificialThreadPrivate(
6098         CGF, CGM.getContext().VoidPtrTy,
6099         generateUniqueName(CGM, "reduction", RCG.getRefExpr(N)));
6100     CGF.Builder.CreateStore(
6101         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6102             RCG.getSharedLValue(N).getPointer(), CGM.VoidPtrTy),
6103         SharedAddr, /*IsVolatile=*/false);
6104   }
6105 }
6106
6107 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6108                                               SourceLocation Loc,
6109                                               llvm::Value *ReductionsPtr,
6110                                               LValue SharedLVal) {
6111   // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6112   // *d);
6113   llvm::Value *Args[] = {
6114       CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy,
6115                                 /*isSigned=*/true),
6116       ReductionsPtr,
6117       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(SharedLVal.getPointer(),
6118                                                       CGM.VoidPtrTy)};
6119   return Address(
6120       CGF.EmitRuntimeCall(
6121           createRuntimeFunction(OMPRTL__kmpc_task_reduction_get_th_data), Args),
6122       SharedLVal.getAlignment());
6123 }
6124
6125 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
6126                                        SourceLocation Loc) {
6127   if (!CGF.HaveInsertPoint())
6128     return;
6129   // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6130   // global_tid);
6131   llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
6132   // Ignore return result until untied tasks are supported.
6133   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
6134   if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
6135     Region->emitUntiedSwitch(CGF);
6136 }
6137
6138 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6139                                            OpenMPDirectiveKind InnerKind,
6140                                            const RegionCodeGenTy &CodeGen,
6141                                            bool HasCancel) {
6142   if (!CGF.HaveInsertPoint())
6143     return;
6144   InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
6145   CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6146 }
6147
6148 namespace {
6149 enum RTCancelKind {
6150   CancelNoreq = 0,
6151   CancelParallel = 1,
6152   CancelLoop = 2,
6153   CancelSections = 3,
6154   CancelTaskgroup = 4
6155 };
6156 } // anonymous namespace
6157
6158 static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6159   RTCancelKind CancelKind = CancelNoreq;
6160   if (CancelRegion == OMPD_parallel)
6161     CancelKind = CancelParallel;
6162   else if (CancelRegion == OMPD_for)
6163     CancelKind = CancelLoop;
6164   else if (CancelRegion == OMPD_sections)
6165     CancelKind = CancelSections;
6166   else {
6167     assert(CancelRegion == OMPD_taskgroup);
6168     CancelKind = CancelTaskgroup;
6169   }
6170   return CancelKind;
6171 }
6172
6173 void CGOpenMPRuntime::emitCancellationPointCall(
6174     CodeGenFunction &CGF, SourceLocation Loc,
6175     OpenMPDirectiveKind CancelRegion) {
6176   if (!CGF.HaveInsertPoint())
6177     return;
6178   // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6179   // global_tid, kmp_int32 cncl_kind);
6180   if (auto *OMPRegionInfo =
6181           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6182     // For 'cancellation point taskgroup', the task region info may not have a
6183     // cancel. This may instead happen in another adjacent task.
6184     if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6185       llvm::Value *Args[] = {
6186           emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6187           CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6188       // Ignore return result until untied tasks are supported.
6189       llvm::Value *Result = CGF.EmitRuntimeCall(
6190           createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
6191       // if (__kmpc_cancellationpoint()) {
6192       //   exit from construct;
6193       // }
6194       llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6195       llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6196       llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6197       CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6198       CGF.EmitBlock(ExitBB);
6199       // exit from construct;
6200       CodeGenFunction::JumpDest CancelDest =
6201           CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6202       CGF.EmitBranchThroughCleanup(CancelDest);
6203       CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6204     }
6205   }
6206 }
6207
6208 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6209                                      const Expr *IfCond,
6210                                      OpenMPDirectiveKind CancelRegion) {
6211   if (!CGF.HaveInsertPoint())
6212     return;
6213   // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6214   // kmp_int32 cncl_kind);
6215   if (auto *OMPRegionInfo =
6216           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
6217     auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF,
6218                                                         PrePostActionTy &) {
6219       CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6220       llvm::Value *Args[] = {
6221           RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6222           CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
6223       // Ignore return result until untied tasks are supported.
6224       llvm::Value *Result = CGF.EmitRuntimeCall(
6225           RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
6226       // if (__kmpc_cancel()) {
6227       //   exit from construct;
6228       // }
6229       llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit");
6230       llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue");
6231       llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6232       CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6233       CGF.EmitBlock(ExitBB);
6234       // exit from construct;
6235       CodeGenFunction::JumpDest CancelDest =
6236           CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6237       CGF.EmitBranchThroughCleanup(CancelDest);
6238       CGF.EmitBlock(ContBB, /*IsFinished=*/true);
6239     };
6240     if (IfCond) {
6241       emitOMPIfClause(CGF, IfCond, ThenGen,
6242                       [](CodeGenFunction &, PrePostActionTy &) {});
6243     } else {
6244       RegionCodeGenTy ThenRCG(ThenGen);
6245       ThenRCG(CGF);
6246     }
6247   }
6248 }
6249
6250 void CGOpenMPRuntime::emitTargetOutlinedFunction(
6251     const OMPExecutableDirective &D, StringRef ParentName,
6252     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6253     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6254   assert(!ParentName.empty() && "Invalid target region parent name!");
6255   emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6256                                    IsOffloadEntry, CodeGen);
6257 }
6258
6259 void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6260     const OMPExecutableDirective &D, StringRef ParentName,
6261     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6262     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6263   // Create a unique name for the entry function using the source location
6264   // information of the current target region. The name will be something like:
6265   //
6266   // __omp_offloading_DD_FFFF_PP_lBB
6267   //
6268   // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the
6269   // mangled name of the function that encloses the target region and BB is the
6270   // line number of the target region.
6271
6272   unsigned DeviceID;
6273   unsigned FileID;
6274   unsigned Line;
6275   getTargetEntryUniqueInfo(CGM.getContext(), D.getLocStart(), DeviceID, FileID,
6276                            Line);
6277   SmallString<64> EntryFnName;
6278   {
6279     llvm::raw_svector_ostream OS(EntryFnName);
6280     OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
6281        << llvm::format("_%x_", FileID) << ParentName << "_l" << Line;
6282   }
6283
6284   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6285
6286   CodeGenFunction CGF(CGM, true);
6287   CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6288   CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6289
6290   OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
6291
6292   // If this target outline function is not an offload entry, we don't need to
6293   // register it.
6294   if (!IsOffloadEntry)
6295     return;
6296
6297   // The target region ID is used by the runtime library to identify the current
6298   // target region, so it only has to be unique and not necessarily point to
6299   // anything. It could be the pointer to the outlined function that implements
6300   // the target region, but we aren't using that so that the compiler doesn't
6301   // need to keep that, and could therefore inline the host function if proven
6302   // worthwhile during optimization. In the other hand, if emitting code for the
6303   // device, the ID has to be the function address so that it can retrieved from
6304   // the offloading entry and launched by the runtime library. We also mark the
6305   // outlined function to have external linkage in case we are emitting code for
6306   // the device, because these functions will be entry points to the device.
6307
6308   if (CGM.getLangOpts().OpenMPIsDevice) {
6309     OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
6310     OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
6311     OutlinedFn->setDSOLocal(false);
6312   } else {
6313     std::string Name = getName({EntryFnName, "region_id"});
6314     OutlinedFnID = new llvm::GlobalVariable(
6315         CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
6316         llvm::GlobalValue::WeakAnyLinkage,
6317         llvm::Constant::getNullValue(CGM.Int8Ty), Name);
6318   }
6319
6320   // Register the information for the entry associated with this target region.
6321   OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
6322       DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
6323       OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion);
6324 }
6325
6326 /// discard all CompoundStmts intervening between two constructs
6327 static const Stmt *ignoreCompoundStmts(const Stmt *Body) {
6328   while (const auto *CS = dyn_cast_or_null<CompoundStmt>(Body))
6329     Body = CS->body_front();
6330
6331   return Body;
6332 }
6333
6334 /// Emit the number of teams for a target directive.  Inspect the num_teams
6335 /// clause associated with a teams construct combined or closely nested
6336 /// with the target directive.
6337 ///
6338 /// Emit a team of size one for directives such as 'target parallel' that
6339 /// have no associated teams construct.
6340 ///
6341 /// Otherwise, return nullptr.
6342 static llvm::Value *
6343 emitNumTeamsForTargetDirective(CGOpenMPRuntime &OMPRuntime,
6344                                CodeGenFunction &CGF,
6345                                const OMPExecutableDirective &D) {
6346   assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
6347                                               "teams directive expected to be "
6348                                               "emitted only for the host!");
6349
6350   CGBuilderTy &Bld = CGF.Builder;
6351
6352   // If the target directive is combined with a teams directive:
6353   //   Return the value in the num_teams clause, if any.
6354   //   Otherwise, return 0 to denote the runtime default.
6355   if (isOpenMPTeamsDirective(D.getDirectiveKind())) {
6356     if (const auto *NumTeamsClause = D.getSingleClause<OMPNumTeamsClause>()) {
6357       CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6358       llvm::Value *NumTeams = CGF.EmitScalarExpr(NumTeamsClause->getNumTeams(),
6359                                                  /*IgnoreResultAssign*/ true);
6360       return Bld.CreateIntCast(NumTeams, CGF.Int32Ty,
6361                                /*IsSigned=*/true);
6362     }
6363
6364     // The default value is 0.
6365     return Bld.getInt32(0);
6366   }
6367
6368   // If the target directive is combined with a parallel directive but not a
6369   // teams directive, start one team.
6370   if (isOpenMPParallelDirective(D.getDirectiveKind()))
6371     return Bld.getInt32(1);
6372
6373   // If the current target region has a teams region enclosed, we need to get
6374   // the number of teams to pass to the runtime function call. This is done
6375   // by generating the expression in a inlined region. This is required because
6376   // the expression is captured in the enclosing target environment when the
6377   // teams directive is not combined with target.
6378
6379   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6380
6381   if (const auto *TeamsDir = dyn_cast_or_null<OMPExecutableDirective>(
6382           ignoreCompoundStmts(CS.getCapturedStmt()))) {
6383     if (isOpenMPTeamsDirective(TeamsDir->getDirectiveKind())) {
6384       if (const auto *NTE = TeamsDir->getSingleClause<OMPNumTeamsClause>()) {
6385         CGOpenMPInnerExprInfo CGInfo(CGF, CS);
6386         CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6387         llvm::Value *NumTeams = CGF.EmitScalarExpr(NTE->getNumTeams());
6388         return Bld.CreateIntCast(NumTeams, CGF.Int32Ty,
6389                                  /*IsSigned=*/true);
6390       }
6391
6392       // If we have an enclosed teams directive but no num_teams clause we use
6393       // the default value 0.
6394       return Bld.getInt32(0);
6395     }
6396   }
6397
6398   // No teams associated with the directive.
6399   return nullptr;
6400 }
6401
6402 /// Emit the number of threads for a target directive.  Inspect the
6403 /// thread_limit clause associated with a teams construct combined or closely
6404 /// nested with the target directive.
6405 ///
6406 /// Emit the num_threads clause for directives such as 'target parallel' that
6407 /// have no associated teams construct.
6408 ///
6409 /// Otherwise, return nullptr.
6410 static llvm::Value *
6411 emitNumThreadsForTargetDirective(CGOpenMPRuntime &OMPRuntime,
6412                                  CodeGenFunction &CGF,
6413                                  const OMPExecutableDirective &D) {
6414   assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the "
6415                                               "teams directive expected to be "
6416                                               "emitted only for the host!");
6417
6418   CGBuilderTy &Bld = CGF.Builder;
6419
6420   //
6421   // If the target directive is combined with a teams directive:
6422   //   Return the value in the thread_limit clause, if any.
6423   //
6424   // If the target directive is combined with a parallel directive:
6425   //   Return the value in the num_threads clause, if any.
6426   //
6427   // If both clauses are set, select the minimum of the two.
6428   //
6429   // If neither teams or parallel combined directives set the number of threads
6430   // in a team, return 0 to denote the runtime default.
6431   //
6432   // If this is not a teams directive return nullptr.
6433
6434   if (isOpenMPTeamsDirective(D.getDirectiveKind()) ||
6435       isOpenMPParallelDirective(D.getDirectiveKind())) {
6436     llvm::Value *DefaultThreadLimitVal = Bld.getInt32(0);
6437     llvm::Value *NumThreadsVal = nullptr;
6438     llvm::Value *ThreadLimitVal = nullptr;
6439
6440     if (const auto *ThreadLimitClause =
6441             D.getSingleClause<OMPThreadLimitClause>()) {
6442       CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6443       llvm::Value *ThreadLimit =
6444           CGF.EmitScalarExpr(ThreadLimitClause->getThreadLimit(),
6445                              /*IgnoreResultAssign*/ true);
6446       ThreadLimitVal = Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty,
6447                                          /*IsSigned=*/true);
6448     }
6449
6450     if (const auto *NumThreadsClause =
6451             D.getSingleClause<OMPNumThreadsClause>()) {
6452       CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6453       llvm::Value *NumThreads =
6454           CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
6455                              /*IgnoreResultAssign*/ true);
6456       NumThreadsVal =
6457           Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*IsSigned=*/true);
6458     }
6459
6460     // Select the lesser of thread_limit and num_threads.
6461     if (NumThreadsVal)
6462       ThreadLimitVal = ThreadLimitVal
6463                            ? Bld.CreateSelect(Bld.CreateICmpSLT(NumThreadsVal,
6464                                                                 ThreadLimitVal),
6465                                               NumThreadsVal, ThreadLimitVal)
6466                            : NumThreadsVal;
6467
6468     // Set default value passed to the runtime if either teams or a target
6469     // parallel type directive is found but no clause is specified.
6470     if (!ThreadLimitVal)
6471       ThreadLimitVal = DefaultThreadLimitVal;
6472
6473     return ThreadLimitVal;
6474   }
6475
6476   // If the current target region has a teams region enclosed, we need to get
6477   // the thread limit to pass to the runtime function call. This is done
6478   // by generating the expression in a inlined region. This is required because
6479   // the expression is captured in the enclosing target environment when the
6480   // teams directive is not combined with target.
6481
6482   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
6483
6484   if (const auto *TeamsDir = dyn_cast_or_null<OMPExecutableDirective>(
6485           ignoreCompoundStmts(CS.getCapturedStmt()))) {
6486     if (isOpenMPTeamsDirective(TeamsDir->getDirectiveKind())) {
6487       if (const auto *TLE = TeamsDir->getSingleClause<OMPThreadLimitClause>()) {
6488         CGOpenMPInnerExprInfo CGInfo(CGF, CS);
6489         CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6490         llvm::Value *ThreadLimit = CGF.EmitScalarExpr(TLE->getThreadLimit());
6491         return CGF.Builder.CreateIntCast(ThreadLimit, CGF.Int32Ty,
6492                                          /*IsSigned=*/true);
6493       }
6494
6495       // If we have an enclosed teams directive but no thread_limit clause we
6496       // use the default value 0.
6497       return CGF.Builder.getInt32(0);
6498     }
6499   }
6500
6501   // No teams associated with the directive.
6502   return nullptr;
6503 }
6504
6505 namespace {
6506 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
6507
6508 // Utility to handle information from clauses associated with a given
6509 // construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
6510 // It provides a convenient interface to obtain the information and generate
6511 // code for that information.
6512 class MappableExprsHandler {
6513 public:
6514   /// Values for bit flags used to specify the mapping type for
6515   /// offloading.
6516   enum OpenMPOffloadMappingFlags : uint64_t {
6517     /// No flags
6518     OMP_MAP_NONE = 0x0,
6519     /// Allocate memory on the device and move data from host to device.
6520     OMP_MAP_TO = 0x01,
6521     /// Allocate memory on the device and move data from device to host.
6522     OMP_MAP_FROM = 0x02,
6523     /// Always perform the requested mapping action on the element, even
6524     /// if it was already mapped before.
6525     OMP_MAP_ALWAYS = 0x04,
6526     /// Delete the element from the device environment, ignoring the
6527     /// current reference count associated with the element.
6528     OMP_MAP_DELETE = 0x08,
6529     /// The element being mapped is a pointer-pointee pair; both the
6530     /// pointer and the pointee should be mapped.
6531     OMP_MAP_PTR_AND_OBJ = 0x10,
6532     /// This flags signals that the base address of an entry should be
6533     /// passed to the target kernel as an argument.
6534     OMP_MAP_TARGET_PARAM = 0x20,
6535     /// Signal that the runtime library has to return the device pointer
6536     /// in the current position for the data being mapped. Used when we have the
6537     /// use_device_ptr clause.
6538     OMP_MAP_RETURN_PARAM = 0x40,
6539     /// This flag signals that the reference being passed is a pointer to
6540     /// private data.
6541     OMP_MAP_PRIVATE = 0x80,
6542     /// Pass the element to the device by value.
6543     OMP_MAP_LITERAL = 0x100,
6544     /// Implicit map
6545     OMP_MAP_IMPLICIT = 0x200,
6546     /// The 16 MSBs of the flags indicate whether the entry is member of some
6547     /// struct/class.
6548     OMP_MAP_MEMBER_OF = 0xffff000000000000,
6549     LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ OMP_MAP_MEMBER_OF),
6550   };
6551
6552   /// Class that associates information with a base pointer to be passed to the
6553   /// runtime library.
6554   class BasePointerInfo {
6555     /// The base pointer.
6556     llvm::Value *Ptr = nullptr;
6557     /// The base declaration that refers to this device pointer, or null if
6558     /// there is none.
6559     const ValueDecl *DevPtrDecl = nullptr;
6560
6561   public:
6562     BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr)
6563         : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
6564     llvm::Value *operator*() const { return Ptr; }
6565     const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; }
6566     void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; }
6567   };
6568
6569   using MapBaseValuesArrayTy = SmallVector<BasePointerInfo, 4>;
6570   using MapValuesArrayTy = SmallVector<llvm::Value *, 4>;
6571   using MapFlagsArrayTy = SmallVector<OpenMPOffloadMappingFlags, 4>;
6572
6573   /// Map between a struct and the its lowest & highest elements which have been
6574   /// mapped.
6575   /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
6576   ///                    HE(FieldIndex, Pointer)}
6577   struct StructRangeInfoTy {
6578     std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
6579         0, Address::invalid()};
6580     std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
6581         0, Address::invalid()};
6582     Address Base = Address::invalid();
6583   };
6584
6585 private:
6586   /// Kind that defines how a device pointer has to be returned.
6587   struct MapInfo {
6588     OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
6589     OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
6590     OpenMPMapClauseKind MapTypeModifier = OMPC_MAP_unknown;
6591     bool ReturnDevicePointer = false;
6592     bool IsImplicit = false;
6593
6594     MapInfo() = default;
6595     MapInfo(
6596         OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
6597         OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapTypeModifier,
6598         bool ReturnDevicePointer, bool IsImplicit)
6599         : Components(Components), MapType(MapType),
6600           MapTypeModifier(MapTypeModifier),
6601           ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
6602   };
6603
6604   /// If use_device_ptr is used on a pointer which is a struct member and there
6605   /// is no map information about it, then emission of that entry is deferred
6606   /// until the whole struct has been processed.
6607   struct DeferredDevicePtrEntryTy {
6608     const Expr *IE = nullptr;
6609     const ValueDecl *VD = nullptr;
6610
6611     DeferredDevicePtrEntryTy(const Expr *IE, const ValueDecl *VD)
6612         : IE(IE), VD(VD) {}
6613   };
6614
6615   /// Directive from where the map clauses were extracted.
6616   const OMPExecutableDirective &CurDir;
6617
6618   /// Function the directive is being generated for.
6619   CodeGenFunction &CGF;
6620
6621   /// Set of all first private variables in the current directive.
6622   llvm::SmallPtrSet<const VarDecl *, 8> FirstPrivateDecls;
6623
6624   /// Map between device pointer declarations and their expression components.
6625   /// The key value for declarations in 'this' is null.
6626   llvm::DenseMap<
6627       const ValueDecl *,
6628       SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
6629       DevPointersMap;
6630
6631   llvm::Value *getExprTypeSize(const Expr *E) const {
6632     QualType ExprTy = E->getType().getCanonicalType();
6633
6634     // Reference types are ignored for mapping purposes.
6635     if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
6636       ExprTy = RefTy->getPointeeType().getCanonicalType();
6637
6638     // Given that an array section is considered a built-in type, we need to
6639     // do the calculation based on the length of the section instead of relying
6640     // on CGF.getTypeSize(E->getType()).
6641     if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
6642       QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(
6643                             OAE->getBase()->IgnoreParenImpCasts())
6644                             .getCanonicalType();
6645
6646       // If there is no length associated with the expression, that means we
6647       // are using the whole length of the base.
6648       if (!OAE->getLength() && OAE->getColonLoc().isValid())
6649         return CGF.getTypeSize(BaseTy);
6650
6651       llvm::Value *ElemSize;
6652       if (const auto *PTy = BaseTy->getAs<PointerType>()) {
6653         ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType());
6654       } else {
6655         const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr());
6656         assert(ATy && "Expecting array type if not a pointer type.");
6657         ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType());
6658       }
6659
6660       // If we don't have a length at this point, that is because we have an
6661       // array section with a single element.
6662       if (!OAE->getLength())
6663         return ElemSize;
6664
6665       llvm::Value *LengthVal = CGF.EmitScalarExpr(OAE->getLength());
6666       LengthVal =
6667           CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false);
6668       return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
6669     }
6670     return CGF.getTypeSize(ExprTy);
6671   }
6672
6673   /// Return the corresponding bits for a given map clause modifier. Add
6674   /// a flag marking the map as a pointer if requested. Add a flag marking the
6675   /// map as the first one of a series of maps that relate to the same map
6676   /// expression.
6677   OpenMPOffloadMappingFlags getMapTypeBits(OpenMPMapClauseKind MapType,
6678                                            OpenMPMapClauseKind MapTypeModifier,
6679                                            bool IsImplicit, bool AddPtrFlag,
6680                                            bool AddIsTargetParamFlag) const {
6681     OpenMPOffloadMappingFlags Bits =
6682         IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
6683     switch (MapType) {
6684     case OMPC_MAP_alloc:
6685     case OMPC_MAP_release:
6686       // alloc and release is the default behavior in the runtime library,  i.e.
6687       // if we don't pass any bits alloc/release that is what the runtime is
6688       // going to do. Therefore, we don't need to signal anything for these two
6689       // type modifiers.
6690       break;
6691     case OMPC_MAP_to:
6692       Bits |= OMP_MAP_TO;
6693       break;
6694     case OMPC_MAP_from:
6695       Bits |= OMP_MAP_FROM;
6696       break;
6697     case OMPC_MAP_tofrom:
6698       Bits |= OMP_MAP_TO | OMP_MAP_FROM;
6699       break;
6700     case OMPC_MAP_delete:
6701       Bits |= OMP_MAP_DELETE;
6702       break;
6703     case OMPC_MAP_always:
6704     case OMPC_MAP_unknown:
6705       llvm_unreachable("Unexpected map type!");
6706     }
6707     if (AddPtrFlag)
6708       Bits |= OMP_MAP_PTR_AND_OBJ;
6709     if (AddIsTargetParamFlag)
6710       Bits |= OMP_MAP_TARGET_PARAM;
6711     if (MapTypeModifier == OMPC_MAP_always)
6712       Bits |= OMP_MAP_ALWAYS;
6713     return Bits;
6714   }
6715
6716   /// Return true if the provided expression is a final array section. A
6717   /// final array section, is one whose length can't be proved to be one.
6718   bool isFinalArraySectionExpression(const Expr *E) const {
6719     const auto *OASE = dyn_cast<OMPArraySectionExpr>(E);
6720
6721     // It is not an array section and therefore not a unity-size one.
6722     if (!OASE)
6723       return false;
6724
6725     // An array section with no colon always refer to a single element.
6726     if (OASE->getColonLoc().isInvalid())
6727       return false;
6728
6729     const Expr *Length = OASE->getLength();
6730
6731     // If we don't have a length we have to check if the array has size 1
6732     // for this dimension. Also, we should always expect a length if the
6733     // base type is pointer.
6734     if (!Length) {
6735       QualType BaseQTy = OMPArraySectionExpr::getBaseOriginalType(
6736                              OASE->getBase()->IgnoreParenImpCasts())
6737                              .getCanonicalType();
6738       if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
6739         return ATy->getSize().getSExtValue() != 1;
6740       // If we don't have a constant dimension length, we have to consider
6741       // the current section as having any size, so it is not necessarily
6742       // unitary. If it happen to be unity size, that's user fault.
6743       return true;
6744     }
6745
6746     // Check if the length evaluates to 1.
6747     llvm::APSInt ConstLength;
6748     if (!Length->EvaluateAsInt(ConstLength, CGF.getContext()))
6749       return true; // Can have more that size 1.
6750
6751     return ConstLength.getSExtValue() != 1;
6752   }
6753
6754   /// Generate the base pointers, section pointers, sizes and map type
6755   /// bits for the provided map type, map modifier, and expression components.
6756   /// \a IsFirstComponent should be set to true if the provided set of
6757   /// components is the first associated with a capture.
6758   void generateInfoForComponentList(
6759       OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapTypeModifier,
6760       OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
6761       MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
6762       MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
6763       StructRangeInfoTy &PartialStruct, bool IsFirstComponentList,
6764       bool IsImplicit) const {
6765     // The following summarizes what has to be generated for each map and the
6766     // types below. The generated information is expressed in this order:
6767     // base pointer, section pointer, size, flags
6768     // (to add to the ones that come from the map type and modifier).
6769     //
6770     // double d;
6771     // int i[100];
6772     // float *p;
6773     //
6774     // struct S1 {
6775     //   int i;
6776     //   float f[50];
6777     // }
6778     // struct S2 {
6779     //   int i;
6780     //   float f[50];
6781     //   S1 s;
6782     //   double *p;
6783     //   struct S2 *ps;
6784     // }
6785     // S2 s;
6786     // S2 *ps;
6787     //
6788     // map(d)
6789     // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
6790     //
6791     // map(i)
6792     // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
6793     //
6794     // map(i[1:23])
6795     // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
6796     //
6797     // map(p)
6798     // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
6799     //
6800     // map(p[1:24])
6801     // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM
6802     //
6803     // map(s)
6804     // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
6805     //
6806     // map(s.i)
6807     // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
6808     //
6809     // map(s.s.f)
6810     // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
6811     //
6812     // map(s.p)
6813     // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
6814     //
6815     // map(to: s.p[:22])
6816     // &s, &(s.p), sizeof(double*), TARGET_PARAM (*)
6817     // &s, &(s.p), sizeof(double*), MEMBER_OF(1) (**)
6818     // &(s.p), &(s.p[0]), 22*sizeof(double),
6819     //   MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
6820     // (*) alloc space for struct members, only this is a target parameter
6821     // (**) map the pointer (nothing to be mapped in this example) (the compiler
6822     //      optimizes this entry out, same in the examples below)
6823     // (***) map the pointee (map: to)
6824     //
6825     // map(s.ps)
6826     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
6827     //
6828     // map(from: s.ps->s.i)
6829     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6830     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6831     // &(s.ps), &(s.ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ  | FROM
6832     //
6833     // map(to: s.ps->ps)
6834     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6835     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6836     // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ  | TO
6837     //
6838     // map(s.ps->ps->ps)
6839     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6840     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6841     // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
6842     // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
6843     //
6844     // map(to: s.ps->ps->s.f[:22])
6845     // &s, &(s.ps), sizeof(S2*), TARGET_PARAM
6846     // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1)
6847     // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
6848     // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
6849     //
6850     // map(ps)
6851     // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
6852     //
6853     // map(ps->i)
6854     // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
6855     //
6856     // map(ps->s.f)
6857     // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
6858     //
6859     // map(from: ps->p)
6860     // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
6861     //
6862     // map(to: ps->p[:22])
6863     // ps, &(ps->p), sizeof(double*), TARGET_PARAM
6864     // ps, &(ps->p), sizeof(double*), MEMBER_OF(1)
6865     // &(ps->p), &(ps->p[0]), 22*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | TO
6866     //
6867     // map(ps->ps)
6868     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
6869     //
6870     // map(from: ps->ps->s.i)
6871     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
6872     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
6873     // &(ps->ps), &(ps->ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM
6874     //
6875     // map(from: ps->ps->ps)
6876     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
6877     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
6878     // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | FROM
6879     //
6880     // map(ps->ps->ps->ps)
6881     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
6882     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
6883     // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
6884     // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM
6885     //
6886     // map(to: ps->ps->ps->s.f[:22])
6887     // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM
6888     // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1)
6889     // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ
6890     // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO
6891     //
6892     // map(to: s.f[:22]) map(from: s.p[:33])
6893     // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1) +
6894     //     sizeof(double*) (**), TARGET_PARAM
6895     // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
6896     // &s, &(s.p), sizeof(double*), MEMBER_OF(1)
6897     // &(s.p), &(s.p[0]), 33*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | FROM
6898     // (*) allocate contiguous space needed to fit all mapped members even if
6899     //     we allocate space for members not mapped (in this example,
6900     //     s.f[22..49] and s.s are not mapped, yet we must allocate space for
6901     //     them as well because they fall between &s.f[0] and &s.p)
6902     //
6903     // map(from: s.f[:22]) map(to: ps->p[:33])
6904     // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
6905     // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
6906     // ps, &(ps->p), sizeof(double*), MEMBER_OF(2) (*)
6907     // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(2) | PTR_AND_OBJ | TO
6908     // (*) the struct this entry pertains to is the 2nd element in the list of
6909     //     arguments, hence MEMBER_OF(2)
6910     //
6911     // map(from: s.f[:22], s.s) map(to: ps->p[:33])
6912     // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1), TARGET_PARAM
6913     // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
6914     // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
6915     // ps, &(ps->p), sizeof(S2*), TARGET_PARAM
6916     // ps, &(ps->p), sizeof(double*), MEMBER_OF(4) (*)
6917     // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO
6918     // (*) the struct this entry pertains to is the 4th element in the list
6919     //     of arguments, hence MEMBER_OF(4)
6920
6921     // Track if the map information being generated is the first for a capture.
6922     bool IsCaptureFirstInfo = IsFirstComponentList;
6923     bool IsLink = false; // Is this variable a "declare target link"?
6924
6925     // Scan the components from the base to the complete expression.
6926     auto CI = Components.rbegin();
6927     auto CE = Components.rend();
6928     auto I = CI;
6929
6930     // Track if the map information being generated is the first for a list of
6931     // components.
6932     bool IsExpressionFirstInfo = true;
6933     Address BP = Address::invalid();
6934
6935     if (isa<MemberExpr>(I->getAssociatedExpression())) {
6936       // The base is the 'this' pointer. The content of the pointer is going
6937       // to be the base of the field being mapped.
6938       BP = CGF.LoadCXXThisAddress();
6939     } else {
6940       // The base is the reference to the variable.
6941       // BP = &Var.
6942       BP = CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getAddress();
6943       if (const auto *VD =
6944               dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
6945         if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
6946                 isDeclareTargetDeclaration(VD))
6947           if (*Res == OMPDeclareTargetDeclAttr::MT_Link) {
6948             IsLink = true;
6949             BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetLink(VD);
6950           }
6951       }
6952
6953       // If the variable is a pointer and is being dereferenced (i.e. is not
6954       // the last component), the base has to be the pointer itself, not its
6955       // reference. References are ignored for mapping purposes.
6956       QualType Ty =
6957           I->getAssociatedDeclaration()->getType().getNonReferenceType();
6958       if (Ty->isAnyPointerType() && std::next(I) != CE) {
6959         BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>());
6960
6961         // We do not need to generate individual map information for the
6962         // pointer, it can be associated with the combined storage.
6963         ++I;
6964       }
6965     }
6966
6967     // Track whether a component of the list should be marked as MEMBER_OF some
6968     // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
6969     // in a component list should be marked as MEMBER_OF, all subsequent entries
6970     // do not belong to the base struct. E.g.
6971     // struct S2 s;
6972     // s.ps->ps->ps->f[:]
6973     //   (1) (2) (3) (4)
6974     // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
6975     // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
6976     // is the pointee of ps(2) which is not member of struct s, so it should not
6977     // be marked as such (it is still PTR_AND_OBJ).
6978     // The variable is initialized to false so that PTR_AND_OBJ entries which
6979     // are not struct members are not considered (e.g. array of pointers to
6980     // data).
6981     bool ShouldBeMemberOf = false;
6982
6983     // Variable keeping track of whether or not we have encountered a component
6984     // in the component list which is a member expression. Useful when we have a
6985     // pointer or a final array section, in which case it is the previous
6986     // component in the list which tells us whether we have a member expression.
6987     // E.g. X.f[:]
6988     // While processing the final array section "[:]" it is "f" which tells us
6989     // whether we are dealing with a member of a declared struct.
6990     const MemberExpr *EncounteredME = nullptr;
6991
6992     for (; I != CE; ++I) {
6993       // If the current component is member of a struct (parent struct) mark it.
6994       if (!EncounteredME) {
6995         EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());
6996         // If we encounter a PTR_AND_OBJ entry from now on it should be marked
6997         // as MEMBER_OF the parent struct.
6998         if (EncounteredME)
6999           ShouldBeMemberOf = true;
7000       }
7001
7002       auto Next = std::next(I);
7003
7004       // We need to generate the addresses and sizes if this is the last
7005       // component, if the component is a pointer or if it is an array section
7006       // whose length can't be proved to be one. If this is a pointer, it
7007       // becomes the base address for the following components.
7008
7009       // A final array section, is one whose length can't be proved to be one.
7010       bool IsFinalArraySection =
7011           isFinalArraySectionExpression(I->getAssociatedExpression());
7012
7013       // Get information on whether the element is a pointer. Have to do a
7014       // special treatment for array sections given that they are built-in
7015       // types.
7016       const auto *OASE =
7017           dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression());
7018       bool IsPointer =
7019           (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE)
7020                        .getCanonicalType()
7021                        ->isAnyPointerType()) ||
7022           I->getAssociatedExpression()->getType()->isAnyPointerType();
7023
7024       if (Next == CE || IsPointer || IsFinalArraySection) {
7025         // If this is not the last component, we expect the pointer to be
7026         // associated with an array expression or member expression.
7027         assert((Next == CE ||
7028                 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7029                 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7030                 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&
7031                "Unexpected expression");
7032
7033         Address LB =
7034             CGF.EmitOMPSharedLValue(I->getAssociatedExpression()).getAddress();
7035         llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
7036
7037         // If this component is a pointer inside the base struct then we don't
7038         // need to create any entry for it - it will be combined with the object
7039         // it is pointing to into a single PTR_AND_OBJ entry.
7040         bool IsMemberPointer =
7041             IsPointer && EncounteredME &&
7042             (dyn_cast<MemberExpr>(I->getAssociatedExpression()) ==
7043              EncounteredME);
7044         if (!IsMemberPointer) {
7045           BasePointers.push_back(BP.getPointer());
7046           Pointers.push_back(LB.getPointer());
7047           Sizes.push_back(Size);
7048
7049           // We need to add a pointer flag for each map that comes from the
7050           // same expression except for the first one. We also need to signal
7051           // this map is the first one that relates with the current capture
7052           // (there is a set of entries for each capture).
7053           OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7054               MapType, MapTypeModifier, IsImplicit,
7055               !IsExpressionFirstInfo || IsLink, IsCaptureFirstInfo && !IsLink);
7056
7057           if (!IsExpressionFirstInfo) {
7058             // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
7059             // then we reset the TO/FROM/ALWAYS/DELETE flags.
7060             if (IsPointer)
7061               Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS |
7062                          OMP_MAP_DELETE);
7063
7064             if (ShouldBeMemberOf) {
7065               // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
7066               // should be later updated with the correct value of MEMBER_OF.
7067               Flags |= OMP_MAP_MEMBER_OF;
7068               // From now on, all subsequent PTR_AND_OBJ entries should not be
7069               // marked as MEMBER_OF.
7070               ShouldBeMemberOf = false;
7071             }
7072           }
7073
7074           Types.push_back(Flags);
7075         }
7076
7077         // If we have encountered a member expression so far, keep track of the
7078         // mapped member. If the parent is "*this", then the value declaration
7079         // is nullptr.
7080         if (EncounteredME) {
7081           const auto *FD = dyn_cast<FieldDecl>(EncounteredME->getMemberDecl());
7082           unsigned FieldIndex = FD->getFieldIndex();
7083
7084           // Update info about the lowest and highest elements for this struct
7085           if (!PartialStruct.Base.isValid()) {
7086             PartialStruct.LowestElem = {FieldIndex, LB};
7087             PartialStruct.HighestElem = {FieldIndex, LB};
7088             PartialStruct.Base = BP;
7089           } else if (FieldIndex < PartialStruct.LowestElem.first) {
7090             PartialStruct.LowestElem = {FieldIndex, LB};
7091           } else if (FieldIndex > PartialStruct.HighestElem.first) {
7092             PartialStruct.HighestElem = {FieldIndex, LB};
7093           }
7094         }
7095
7096         // If we have a final array section, we are done with this expression.
7097         if (IsFinalArraySection)
7098           break;
7099
7100         // The pointer becomes the base for the next element.
7101         if (Next != CE)
7102           BP = LB;
7103
7104         IsExpressionFirstInfo = false;
7105         IsCaptureFirstInfo = false;
7106       }
7107     }
7108   }
7109
7110   /// Return the adjusted map modifiers if the declaration a capture refers to
7111   /// appears in a first-private clause. This is expected to be used only with
7112   /// directives that start with 'target'.
7113   MappableExprsHandler::OpenMPOffloadMappingFlags
7114   getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
7115     assert(Cap.capturesVariable() && "Expected capture by reference only!");
7116
7117     // A first private variable captured by reference will use only the
7118     // 'private ptr' and 'map to' flag. Return the right flags if the captured
7119     // declaration is known as first-private in this handler.
7120     if (FirstPrivateDecls.count(Cap.getCapturedVar()))
7121       return MappableExprsHandler::OMP_MAP_PRIVATE |
7122              MappableExprsHandler::OMP_MAP_TO;
7123     return MappableExprsHandler::OMP_MAP_TO |
7124            MappableExprsHandler::OMP_MAP_FROM;
7125   }
7126
7127   static OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position) {
7128     // Member of is given by the 16 MSB of the flag, so rotate by 48 bits.
7129     return static_cast<OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
7130                                                   << 48);
7131   }
7132
7133   static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags,
7134                                      OpenMPOffloadMappingFlags MemberOfFlag) {
7135     // If the entry is PTR_AND_OBJ but has not been marked with the special
7136     // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
7137     // marked as MEMBER_OF.
7138     if ((Flags & OMP_MAP_PTR_AND_OBJ) &&
7139         ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF))
7140       return;
7141
7142     // Reset the placeholder value to prepare the flag for the assignment of the
7143     // proper MEMBER_OF value.
7144     Flags &= ~OMP_MAP_MEMBER_OF;
7145     Flags |= MemberOfFlag;
7146   }
7147
7148 public:
7149   MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
7150       : CurDir(Dir), CGF(CGF) {
7151     // Extract firstprivate clause information.
7152     for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
7153       for (const auto *D : C->varlists())
7154         FirstPrivateDecls.insert(
7155             cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
7156     // Extract device pointer clause information.
7157     for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
7158       for (auto L : C->component_lists())
7159         DevPointersMap[L.first].push_back(L.second);
7160   }
7161
7162   /// Generate code for the combined entry if we have a partially mapped struct
7163   /// and take care of the mapping flags of the arguments corresponding to
7164   /// individual struct members.
7165   void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers,
7166                          MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7167                          MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes,
7168                          const StructRangeInfoTy &PartialStruct) const {
7169     // Base is the base of the struct
7170     BasePointers.push_back(PartialStruct.Base.getPointer());
7171     // Pointer is the address of the lowest element
7172     llvm::Value *LB = PartialStruct.LowestElem.second.getPointer();
7173     Pointers.push_back(LB);
7174     // Size is (addr of {highest+1} element) - (addr of lowest element)
7175     llvm::Value *HB = PartialStruct.HighestElem.second.getPointer();
7176     llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1);
7177     llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
7178     llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy);
7179     llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr);
7180     llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.SizeTy,
7181                                                   /*isSinged=*/false);
7182     Sizes.push_back(Size);
7183     // Map type is always TARGET_PARAM
7184     Types.push_back(OMP_MAP_TARGET_PARAM);
7185     // Remove TARGET_PARAM flag from the first element
7186     (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
7187
7188     // All other current entries will be MEMBER_OF the combined entry
7189     // (except for PTR_AND_OBJ entries which do not have a placeholder value
7190     // 0xFFFF in the MEMBER_OF field).
7191     OpenMPOffloadMappingFlags MemberOfFlag =
7192         getMemberOfFlag(BasePointers.size() - 1);
7193     for (auto &M : CurTypes)
7194       setCorrectMemberOfFlag(M, MemberOfFlag);
7195   }
7196
7197   /// Generate all the base pointers, section pointers, sizes and map
7198   /// types for the extracted mappable expressions. Also, for each item that
7199   /// relates with a device pointer, a pair of the relevant declaration and
7200   /// index where it occurs is appended to the device pointers info array.
7201   void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
7202                        MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7203                        MapFlagsArrayTy &Types) const {
7204     // We have to process the component lists that relate with the same
7205     // declaration in a single chunk so that we can generate the map flags
7206     // correctly. Therefore, we organize all lists in a map.
7207     llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
7208
7209     // Helper function to fill the information map for the different supported
7210     // clauses.
7211     auto &&InfoGen = [&Info](
7212         const ValueDecl *D,
7213         OMPClauseMappableExprCommon::MappableExprComponentListRef L,
7214         OpenMPMapClauseKind MapType, OpenMPMapClauseKind MapModifier,
7215         bool ReturnDevicePointer, bool IsImplicit) {
7216       const ValueDecl *VD =
7217           D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr;
7218       Info[VD].emplace_back(L, MapType, MapModifier, ReturnDevicePointer,
7219                             IsImplicit);
7220     };
7221
7222     // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
7223     for (const auto *C : this->CurDir.getClausesOfKind<OMPMapClause>())
7224       for (const auto &L : C->component_lists()) {
7225         InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifier(),
7226             /*ReturnDevicePointer=*/false, C->isImplicit());
7227       }
7228     for (const auto *C : this->CurDir.getClausesOfKind<OMPToClause>())
7229       for (const auto &L : C->component_lists()) {
7230         InfoGen(L.first, L.second, OMPC_MAP_to, OMPC_MAP_unknown,
7231             /*ReturnDevicePointer=*/false, C->isImplicit());
7232       }
7233     for (const auto *C : this->CurDir.getClausesOfKind<OMPFromClause>())
7234       for (const auto &L : C->component_lists()) {
7235         InfoGen(L.first, L.second, OMPC_MAP_from, OMPC_MAP_unknown,
7236             /*ReturnDevicePointer=*/false, C->isImplicit());
7237       }
7238
7239     // Look at the use_device_ptr clause information and mark the existing map
7240     // entries as such. If there is no map information for an entry in the
7241     // use_device_ptr list, we create one with map type 'alloc' and zero size
7242     // section. It is the user fault if that was not mapped before. If there is
7243     // no map information and the pointer is a struct member, then we defer the
7244     // emission of that entry until the whole struct has been processed.
7245     llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
7246         DeferredInfo;
7247
7248     // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
7249     for (const auto *C :
7250         this->CurDir.getClausesOfKind<OMPUseDevicePtrClause>()) {
7251       for (const auto &L : C->component_lists()) {
7252         assert(!L.second.empty() && "Not expecting empty list of components!");
7253         const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
7254         VD = cast<ValueDecl>(VD->getCanonicalDecl());
7255         const Expr *IE = L.second.back().getAssociatedExpression();
7256         // If the first component is a member expression, we have to look into
7257         // 'this', which maps to null in the map of map information. Otherwise
7258         // look directly for the information.
7259         auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);
7260
7261         // We potentially have map information for this declaration already.
7262         // Look for the first set of components that refer to it.
7263         if (It != Info.end()) {
7264           auto CI = std::find_if(
7265               It->second.begin(), It->second.end(), [VD](const MapInfo &MI) {
7266                 return MI.Components.back().getAssociatedDeclaration() == VD;
7267               });
7268           // If we found a map entry, signal that the pointer has to be returned
7269           // and move on to the next declaration.
7270           if (CI != It->second.end()) {
7271             CI->ReturnDevicePointer = true;
7272             continue;
7273           }
7274         }
7275
7276         // We didn't find any match in our map information - generate a zero
7277         // size array section - if the pointer is a struct member we defer this
7278         // action until the whole struct has been processed.
7279         // FIXME: MSVC 2013 seems to require this-> to find member CGF.
7280         if (isa<MemberExpr>(IE)) {
7281           // Insert the pointer into Info to be processed by
7282           // generateInfoForComponentList. Because it is a member pointer
7283           // without a pointee, no entry will be generated for it, therefore
7284           // we need to generate one after the whole struct has been processed.
7285           // Nonetheless, generateInfoForComponentList must be called to take
7286           // the pointer into account for the calculation of the range of the
7287           // partial struct.
7288           InfoGen(nullptr, L.second, OMPC_MAP_unknown, OMPC_MAP_unknown,
7289                   /*ReturnDevicePointer=*/false, C->isImplicit());
7290           DeferredInfo[nullptr].emplace_back(IE, VD);
7291         } else {
7292           llvm::Value *Ptr = this->CGF.EmitLoadOfScalar(
7293               this->CGF.EmitLValue(IE), IE->getExprLoc());
7294           BasePointers.emplace_back(Ptr, VD);
7295           Pointers.push_back(Ptr);
7296           Sizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy));
7297           Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
7298         }
7299       }
7300     }
7301
7302     for (const auto &M : Info) {
7303       // We need to know when we generate information for the first component
7304       // associated with a capture, because the mapping flags depend on it.
7305       bool IsFirstComponentList = true;
7306
7307       // Temporary versions of arrays
7308       MapBaseValuesArrayTy CurBasePointers;
7309       MapValuesArrayTy CurPointers;
7310       MapValuesArrayTy CurSizes;
7311       MapFlagsArrayTy CurTypes;
7312       StructRangeInfoTy PartialStruct;
7313
7314       for (const MapInfo &L : M.second) {
7315         assert(!L.Components.empty() &&
7316                "Not expecting declaration with no component lists.");
7317
7318         // Remember the current base pointer index.
7319         unsigned CurrentBasePointersIdx = CurBasePointers.size();
7320         // FIXME: MSVC 2013 seems to require this-> to find the member method.
7321         this->generateInfoForComponentList(
7322             L.MapType, L.MapTypeModifier, L.Components, CurBasePointers,
7323             CurPointers, CurSizes, CurTypes, PartialStruct,
7324             IsFirstComponentList, L.IsImplicit);
7325
7326         // If this entry relates with a device pointer, set the relevant
7327         // declaration and add the 'return pointer' flag.
7328         if (L.ReturnDevicePointer) {
7329           assert(CurBasePointers.size() > CurrentBasePointersIdx &&
7330                  "Unexpected number of mapped base pointers.");
7331
7332           const ValueDecl *RelevantVD =
7333               L.Components.back().getAssociatedDeclaration();
7334           assert(RelevantVD &&
7335                  "No relevant declaration related with device pointer??");
7336
7337           CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
7338           CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
7339         }
7340         IsFirstComponentList = false;
7341       }
7342
7343       // Append any pending zero-length pointers which are struct members and
7344       // used with use_device_ptr.
7345       auto CI = DeferredInfo.find(M.first);
7346       if (CI != DeferredInfo.end()) {
7347         for (const DeferredDevicePtrEntryTy &L : CI->second) {
7348           llvm::Value *BasePtr = this->CGF.EmitLValue(L.IE).getPointer();
7349           llvm::Value *Ptr = this->CGF.EmitLoadOfScalar(
7350               this->CGF.EmitLValue(L.IE), L.IE->getExprLoc());
7351           CurBasePointers.emplace_back(BasePtr, L.VD);
7352           CurPointers.push_back(Ptr);
7353           CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.SizeTy));
7354           // Entry is PTR_AND_OBJ and RETURN_PARAM. Also, set the placeholder
7355           // value MEMBER_OF=FFFF so that the entry is later updated with the
7356           // correct value of MEMBER_OF.
7357           CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM |
7358                              OMP_MAP_MEMBER_OF);
7359         }
7360       }
7361
7362       // If there is an entry in PartialStruct it means we have a struct with
7363       // individual members mapped. Emit an extra combined entry.
7364       if (PartialStruct.Base.isValid())
7365         emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
7366                           PartialStruct);
7367
7368       // We need to append the results of this capture to what we already have.
7369       BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
7370       Pointers.append(CurPointers.begin(), CurPointers.end());
7371       Sizes.append(CurSizes.begin(), CurSizes.end());
7372       Types.append(CurTypes.begin(), CurTypes.end());
7373     }
7374   }
7375
7376   /// Generate the base pointers, section pointers, sizes and map types
7377   /// associated to a given capture.
7378   void generateInfoForCapture(const CapturedStmt::Capture *Cap,
7379                               llvm::Value *Arg,
7380                               MapBaseValuesArrayTy &BasePointers,
7381                               MapValuesArrayTy &Pointers,
7382                               MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
7383                               StructRangeInfoTy &PartialStruct) const {
7384     assert(!Cap->capturesVariableArrayType() &&
7385            "Not expecting to generate map info for a variable array type!");
7386
7387     // We need to know when we generating information for the first component
7388     // associated with a capture, because the mapping flags depend on it.
7389     bool IsFirstComponentList = true;
7390
7391     const ValueDecl *VD = Cap->capturesThis()
7392                               ? nullptr
7393                               : Cap->getCapturedVar()->getCanonicalDecl();
7394
7395     // If this declaration appears in a is_device_ptr clause we just have to
7396     // pass the pointer by value. If it is a reference to a declaration, we just
7397     // pass its value.
7398     if (DevPointersMap.count(VD)) {
7399       BasePointers.emplace_back(Arg, VD);
7400       Pointers.push_back(Arg);
7401       Sizes.push_back(CGF.getTypeSize(CGF.getContext().VoidPtrTy));
7402       Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
7403       return;
7404     }
7405
7406     // FIXME: MSVC 2013 seems to require this-> to find member CurDir.
7407     for (const auto *C : this->CurDir.getClausesOfKind<OMPMapClause>())
7408       for (const auto &L : C->decl_component_lists(VD)) {
7409         assert(L.first == VD &&
7410                "We got information for the wrong declaration??");
7411         assert(!L.second.empty() &&
7412                "Not expecting declaration with no component lists.");
7413         generateInfoForComponentList(C->getMapType(), C->getMapTypeModifier(),
7414                                      L.second, BasePointers, Pointers, Sizes,
7415                                      Types, PartialStruct, IsFirstComponentList,
7416                                      C->isImplicit());
7417         IsFirstComponentList = false;
7418       }
7419   }
7420
7421   /// Generate the base pointers, section pointers, sizes and map types
7422   /// associated with the declare target link variables.
7423   void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers,
7424                                         MapValuesArrayTy &Pointers,
7425                                         MapValuesArrayTy &Sizes,
7426                                         MapFlagsArrayTy &Types) const {
7427     // Map other list items in the map clause which are not captured variables
7428     // but "declare target link" global variables.,
7429     for (const auto *C : this->CurDir.getClausesOfKind<OMPMapClause>()) {
7430       for (const auto &L : C->component_lists()) {
7431         if (!L.first)
7432           continue;
7433         const auto *VD = dyn_cast<VarDecl>(L.first);
7434         if (!VD)
7435           continue;
7436         llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
7437             isDeclareTargetDeclaration(VD);
7438         if (!Res || *Res != OMPDeclareTargetDeclAttr::MT_Link)
7439           continue;
7440         StructRangeInfoTy PartialStruct;
7441         generateInfoForComponentList(
7442             C->getMapType(), C->getMapTypeModifier(), L.second, BasePointers,
7443             Pointers, Sizes, Types, PartialStruct,
7444             /*IsFirstComponentList=*/true, C->isImplicit());
7445         assert(!PartialStruct.Base.isValid() &&
7446                "No partial structs for declare target link expected.");
7447       }
7448     }
7449   }
7450
7451   /// Generate the default map information for a given capture \a CI,
7452   /// record field declaration \a RI and captured value \a CV.
7453   void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
7454                               const FieldDecl &RI, llvm::Value *CV,
7455                               MapBaseValuesArrayTy &CurBasePointers,
7456                               MapValuesArrayTy &CurPointers,
7457                               MapValuesArrayTy &CurSizes,
7458                               MapFlagsArrayTy &CurMapTypes) const {
7459     // Do the default mapping.
7460     if (CI.capturesThis()) {
7461       CurBasePointers.push_back(CV);
7462       CurPointers.push_back(CV);
7463       const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
7464       CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType()));
7465       // Default map type.
7466       CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
7467     } else if (CI.capturesVariableByCopy()) {
7468       CurBasePointers.push_back(CV);
7469       CurPointers.push_back(CV);
7470       if (!RI.getType()->isAnyPointerType()) {
7471         // We have to signal to the runtime captures passed by value that are
7472         // not pointers.
7473         CurMapTypes.push_back(OMP_MAP_LITERAL);
7474         CurSizes.push_back(CGF.getTypeSize(RI.getType()));
7475       } else {
7476         // Pointers are implicitly mapped with a zero size and no flags
7477         // (other than first map that is added for all implicit maps).
7478         CurMapTypes.push_back(OMP_MAP_NONE);
7479         CurSizes.push_back(llvm::Constant::getNullValue(CGF.SizeTy));
7480       }
7481     } else {
7482       assert(CI.capturesVariable() && "Expected captured reference.");
7483       CurBasePointers.push_back(CV);
7484       CurPointers.push_back(CV);
7485
7486       const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
7487       QualType ElementType = PtrTy->getPointeeType();
7488       CurSizes.push_back(CGF.getTypeSize(ElementType));
7489       // The default map type for a scalar/complex type is 'to' because by
7490       // default the value doesn't have to be retrieved. For an aggregate
7491       // type, the default is 'tofrom'.
7492       CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI));
7493     }
7494     // Every default map produces a single argument which is a target parameter.
7495     CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
7496
7497     // Add flag stating this is an implicit map.
7498     CurMapTypes.back() |= OMP_MAP_IMPLICIT;
7499   }
7500 };
7501
7502 enum OpenMPOffloadingReservedDeviceIDs {
7503   /// Device ID if the device was not defined, runtime should get it
7504   /// from environment variables in the spec.
7505   OMP_DEVICEID_UNDEF = -1,
7506 };
7507 } // anonymous namespace
7508
7509 /// Emit the arrays used to pass the captures and map information to the
7510 /// offloading runtime library. If there is no map or capture information,
7511 /// return nullptr by reference.
7512 static void
7513 emitOffloadingArrays(CodeGenFunction &CGF,
7514                      MappableExprsHandler::MapBaseValuesArrayTy &BasePointers,
7515                      MappableExprsHandler::MapValuesArrayTy &Pointers,
7516                      MappableExprsHandler::MapValuesArrayTy &Sizes,
7517                      MappableExprsHandler::MapFlagsArrayTy &MapTypes,
7518                      CGOpenMPRuntime::TargetDataInfo &Info) {
7519   CodeGenModule &CGM = CGF.CGM;
7520   ASTContext &Ctx = CGF.getContext();
7521
7522   // Reset the array information.
7523   Info.clearArrayInfo();
7524   Info.NumberOfPtrs = BasePointers.size();
7525
7526   if (Info.NumberOfPtrs) {
7527     // Detect if we have any capture size requiring runtime evaluation of the
7528     // size so that a constant array could be eventually used.
7529     bool hasRuntimeEvaluationCaptureSize = false;
7530     for (llvm::Value *S : Sizes)
7531       if (!isa<llvm::Constant>(S)) {
7532         hasRuntimeEvaluationCaptureSize = true;
7533         break;
7534       }
7535
7536     llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true);
7537     QualType PointerArrayType =
7538         Ctx.getConstantArrayType(Ctx.VoidPtrTy, PointerNumAP, ArrayType::Normal,
7539                                  /*IndexTypeQuals=*/0);
7540
7541     Info.BasePointersArray =
7542         CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer();
7543     Info.PointersArray =
7544         CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer();
7545
7546     // If we don't have any VLA types or other types that require runtime
7547     // evaluation, we can use a constant array for the map sizes, otherwise we
7548     // need to fill up the arrays as we do for the pointers.
7549     if (hasRuntimeEvaluationCaptureSize) {
7550       QualType SizeArrayType = Ctx.getConstantArrayType(
7551           Ctx.getSizeType(), PointerNumAP, ArrayType::Normal,
7552           /*IndexTypeQuals=*/0);
7553       Info.SizesArray =
7554           CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer();
7555     } else {
7556       // We expect all the sizes to be constant, so we collect them to create
7557       // a constant array.
7558       SmallVector<llvm::Constant *, 16> ConstSizes;
7559       for (llvm::Value *S : Sizes)
7560         ConstSizes.push_back(cast<llvm::Constant>(S));
7561
7562       auto *SizesArrayInit = llvm::ConstantArray::get(
7563           llvm::ArrayType::get(CGM.SizeTy, ConstSizes.size()), ConstSizes);
7564       std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"});
7565       auto *SizesArrayGbl = new llvm::GlobalVariable(
7566           CGM.getModule(), SizesArrayInit->getType(),
7567           /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
7568           SizesArrayInit, Name);
7569       SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
7570       Info.SizesArray = SizesArrayGbl;
7571     }
7572
7573     // The map types are always constant so we don't need to generate code to
7574     // fill arrays. Instead, we create an array constant.
7575     SmallVector<uint64_t, 4> Mapping(MapTypes.size(), 0);
7576     llvm::copy(MapTypes, Mapping.begin());
7577     llvm::Constant *MapTypesArrayInit =
7578         llvm::ConstantDataArray::get(CGF.Builder.getContext(), Mapping);
7579     std::string MaptypesName =
7580         CGM.getOpenMPRuntime().getName({"offload_maptypes"});
7581     auto *MapTypesArrayGbl = new llvm::GlobalVariable(
7582         CGM.getModule(), MapTypesArrayInit->getType(),
7583         /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage,
7584         MapTypesArrayInit, MaptypesName);
7585     MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
7586     Info.MapTypesArray = MapTypesArrayGbl;
7587
7588     for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
7589       llvm::Value *BPVal = *BasePointers[I];
7590       llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
7591           llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
7592           Info.BasePointersArray, 0, I);
7593       BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
7594           BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0));
7595       Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
7596       CGF.Builder.CreateStore(BPVal, BPAddr);
7597
7598       if (Info.requiresDevicePointerInfo())
7599         if (const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl())
7600           Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
7601
7602       llvm::Value *PVal = Pointers[I];
7603       llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32(
7604           llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
7605           Info.PointersArray, 0, I);
7606       P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
7607           P, PVal->getType()->getPointerTo(/*AddrSpace=*/0));
7608       Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
7609       CGF.Builder.CreateStore(PVal, PAddr);
7610
7611       if (hasRuntimeEvaluationCaptureSize) {
7612         llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32(
7613             llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs),
7614             Info.SizesArray,
7615             /*Idx0=*/0,
7616             /*Idx1=*/I);
7617         Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
7618         CGF.Builder.CreateStore(
7619             CGF.Builder.CreateIntCast(Sizes[I], CGM.SizeTy, /*isSigned=*/true),
7620             SAddr);
7621       }
7622     }
7623   }
7624 }
7625 /// Emit the arguments to be passed to the runtime library based on the
7626 /// arrays of pointers, sizes and map types.
7627 static void emitOffloadingArraysArgument(
7628     CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg,
7629     llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg,
7630     llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
7631   CodeGenModule &CGM = CGF.CGM;
7632   if (Info.NumberOfPtrs) {
7633     BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
7634         llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
7635         Info.BasePointersArray,
7636         /*Idx0=*/0, /*Idx1=*/0);
7637     PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
7638         llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
7639         Info.PointersArray,
7640         /*Idx0=*/0,
7641         /*Idx1=*/0);
7642     SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
7643         llvm::ArrayType::get(CGM.SizeTy, Info.NumberOfPtrs), Info.SizesArray,
7644         /*Idx0=*/0, /*Idx1=*/0);
7645     MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32(
7646         llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs),
7647         Info.MapTypesArray,
7648         /*Idx0=*/0,
7649         /*Idx1=*/0);
7650   } else {
7651     BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
7652     PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
7653     SizesArrayArg = llvm::ConstantPointerNull::get(CGM.SizeTy->getPointerTo());
7654     MapTypesArrayArg =
7655         llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo());
7656   }
7657 }
7658
7659 void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
7660                                      const OMPExecutableDirective &D,
7661                                      llvm::Value *OutlinedFn,
7662                                      llvm::Value *OutlinedFnID,
7663                                      const Expr *IfCond, const Expr *Device) {
7664   if (!CGF.HaveInsertPoint())
7665     return;
7666
7667   assert(OutlinedFn && "Invalid outlined function!");
7668
7669   const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>();
7670   llvm::SmallVector<llvm::Value *, 16> CapturedVars;
7671   const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target);
7672   auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
7673                                             PrePostActionTy &) {
7674     CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
7675   };
7676   emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen);
7677
7678   CodeGenFunction::OMPTargetDataInfo InputInfo;
7679   llvm::Value *MapTypesArray = nullptr;
7680   // Fill up the pointer arrays and transfer execution to the device.
7681   auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
7682                     &MapTypesArray, &CS, RequiresOuterTask,
7683                     &CapturedVars](CodeGenFunction &CGF, PrePostActionTy &) {
7684     // On top of the arrays that were filled up, the target offloading call
7685     // takes as arguments the device id as well as the host pointer. The host
7686     // pointer is used by the runtime library to identify the current target
7687     // region, so it only has to be unique and not necessarily point to
7688     // anything. It could be the pointer to the outlined function that
7689     // implements the target region, but we aren't using that so that the
7690     // compiler doesn't need to keep that, and could therefore inline the host
7691     // function if proven worthwhile during optimization.
7692
7693     // From this point on, we need to have an ID of the target region defined.
7694     assert(OutlinedFnID && "Invalid outlined function ID!");
7695
7696     // Emit device ID if any.
7697     llvm::Value *DeviceID;
7698     if (Device) {
7699       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
7700                                            CGF.Int64Ty, /*isSigned=*/true);
7701     } else {
7702       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
7703     }
7704
7705     // Emit the number of elements in the offloading arrays.
7706     llvm::Value *PointerNum =
7707         CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
7708
7709     // Return value of the runtime offloading call.
7710     llvm::Value *Return;
7711
7712     llvm::Value *NumTeams = emitNumTeamsForTargetDirective(*this, CGF, D);
7713     llvm::Value *NumThreads = emitNumThreadsForTargetDirective(*this, CGF, D);
7714
7715     bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
7716     // The target region is an outlined function launched by the runtime
7717     // via calls __tgt_target() or __tgt_target_teams().
7718     //
7719     // __tgt_target() launches a target region with one team and one thread,
7720     // executing a serial region.  This master thread may in turn launch
7721     // more threads within its team upon encountering a parallel region,
7722     // however, no additional teams can be launched on the device.
7723     //
7724     // __tgt_target_teams() launches a target region with one or more teams,
7725     // each with one or more threads.  This call is required for target
7726     // constructs such as:
7727     //  'target teams'
7728     //  'target' / 'teams'
7729     //  'target teams distribute parallel for'
7730     //  'target parallel'
7731     // and so on.
7732     //
7733     // Note that on the host and CPU targets, the runtime implementation of
7734     // these calls simply call the outlined function without forking threads.
7735     // The outlined functions themselves have runtime calls to
7736     // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
7737     // the compiler in emitTeamsCall() and emitParallelCall().
7738     //
7739     // In contrast, on the NVPTX target, the implementation of
7740     // __tgt_target_teams() launches a GPU kernel with the requested number
7741     // of teams and threads so no additional calls to the runtime are required.
7742     if (NumTeams) {
7743       // If we have NumTeams defined this means that we have an enclosed teams
7744       // region. Therefore we also expect to have NumThreads defined. These two
7745       // values should be defined in the presence of a teams directive,
7746       // regardless of having any clauses associated. If the user is using teams
7747       // but no clauses, these two values will be the default that should be
7748       // passed to the runtime library - a 32-bit integer with the value zero.
7749       assert(NumThreads && "Thread limit expression should be available along "
7750                            "with number of teams.");
7751       llvm::Value *OffloadingArgs[] = {DeviceID,
7752                                        OutlinedFnID,
7753                                        PointerNum,
7754                                        InputInfo.BasePointersArray.getPointer(),
7755                                        InputInfo.PointersArray.getPointer(),
7756                                        InputInfo.SizesArray.getPointer(),
7757                                        MapTypesArray,
7758                                        NumTeams,
7759                                        NumThreads};
7760       Return = CGF.EmitRuntimeCall(
7761           createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_teams_nowait
7762                                           : OMPRTL__tgt_target_teams),
7763           OffloadingArgs);
7764     } else {
7765       llvm::Value *OffloadingArgs[] = {DeviceID,
7766                                        OutlinedFnID,
7767                                        PointerNum,
7768                                        InputInfo.BasePointersArray.getPointer(),
7769                                        InputInfo.PointersArray.getPointer(),
7770                                        InputInfo.SizesArray.getPointer(),
7771                                        MapTypesArray};
7772       Return = CGF.EmitRuntimeCall(
7773           createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_nowait
7774                                           : OMPRTL__tgt_target),
7775           OffloadingArgs);
7776     }
7777
7778     // Check the error code and execute the host version if required.
7779     llvm::BasicBlock *OffloadFailedBlock =
7780         CGF.createBasicBlock("omp_offload.failed");
7781     llvm::BasicBlock *OffloadContBlock =
7782         CGF.createBasicBlock("omp_offload.cont");
7783     llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
7784     CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
7785
7786     CGF.EmitBlock(OffloadFailedBlock);
7787     if (RequiresOuterTask) {
7788       CapturedVars.clear();
7789       CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
7790     }
7791     emitOutlinedFunctionCall(CGF, D.getLocStart(), OutlinedFn, CapturedVars);
7792     CGF.EmitBranch(OffloadContBlock);
7793
7794     CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true);
7795   };
7796
7797   // Notify that the host version must be executed.
7798   auto &&ElseGen = [this, &D, OutlinedFn, &CS, &CapturedVars,
7799                     RequiresOuterTask](CodeGenFunction &CGF,
7800                                        PrePostActionTy &) {
7801     if (RequiresOuterTask) {
7802       CapturedVars.clear();
7803       CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
7804     }
7805     emitOutlinedFunctionCall(CGF, D.getLocStart(), OutlinedFn, CapturedVars);
7806   };
7807
7808   auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
7809                           &CapturedVars, RequiresOuterTask,
7810                           &CS](CodeGenFunction &CGF, PrePostActionTy &) {
7811     // Fill up the arrays with all the captured variables.
7812     MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
7813     MappableExprsHandler::MapValuesArrayTy Pointers;
7814     MappableExprsHandler::MapValuesArrayTy Sizes;
7815     MappableExprsHandler::MapFlagsArrayTy MapTypes;
7816
7817     // Get mappable expression information.
7818     MappableExprsHandler MEHandler(D, CGF);
7819
7820     auto RI = CS.getCapturedRecordDecl()->field_begin();
7821     auto CV = CapturedVars.begin();
7822     for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
7823                                               CE = CS.capture_end();
7824          CI != CE; ++CI, ++RI, ++CV) {
7825       MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers;
7826       MappableExprsHandler::MapValuesArrayTy CurPointers;
7827       MappableExprsHandler::MapValuesArrayTy CurSizes;
7828       MappableExprsHandler::MapFlagsArrayTy CurMapTypes;
7829       MappableExprsHandler::StructRangeInfoTy PartialStruct;
7830
7831       // VLA sizes are passed to the outlined region by copy and do not have map
7832       // information associated.
7833       if (CI->capturesVariableArrayType()) {
7834         CurBasePointers.push_back(*CV);
7835         CurPointers.push_back(*CV);
7836         CurSizes.push_back(CGF.getTypeSize(RI->getType()));
7837         // Copy to the device as an argument. No need to retrieve it.
7838         CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
7839                               MappableExprsHandler::OMP_MAP_TARGET_PARAM);
7840       } else {
7841         // If we have any information in the map clause, we use it, otherwise we
7842         // just do a default mapping.
7843         MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
7844                                          CurSizes, CurMapTypes, PartialStruct);
7845         if (CurBasePointers.empty())
7846           MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
7847                                            CurPointers, CurSizes, CurMapTypes);
7848       }
7849       // We expect to have at least an element of information for this capture.
7850       assert(!CurBasePointers.empty() &&
7851              "Non-existing map pointer for capture!");
7852       assert(CurBasePointers.size() == CurPointers.size() &&
7853              CurBasePointers.size() == CurSizes.size() &&
7854              CurBasePointers.size() == CurMapTypes.size() &&
7855              "Inconsistent map information sizes!");
7856
7857       // If there is an entry in PartialStruct it means we have a struct with
7858       // individual members mapped. Emit an extra combined entry.
7859       if (PartialStruct.Base.isValid())
7860         MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes,
7861                                     CurMapTypes, PartialStruct);
7862
7863       // We need to append the results of this capture to what we already have.
7864       BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
7865       Pointers.append(CurPointers.begin(), CurPointers.end());
7866       Sizes.append(CurSizes.begin(), CurSizes.end());
7867       MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
7868     }
7869     // Map other list items in the map clause which are not captured variables
7870     // but "declare target link" global variables.
7871     MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes,
7872                                                MapTypes);
7873
7874     TargetDataInfo Info;
7875     // Fill up the arrays and create the arguments.
7876     emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
7877     emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
7878                                  Info.PointersArray, Info.SizesArray,
7879                                  Info.MapTypesArray, Info);
7880     InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
7881     InputInfo.BasePointersArray =
7882         Address(Info.BasePointersArray, CGM.getPointerAlign());
7883     InputInfo.PointersArray =
7884         Address(Info.PointersArray, CGM.getPointerAlign());
7885     InputInfo.SizesArray = Address(Info.SizesArray, CGM.getPointerAlign());
7886     MapTypesArray = Info.MapTypesArray;
7887     if (RequiresOuterTask)
7888       CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
7889     else
7890       emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
7891   };
7892
7893   auto &&TargetElseGen = [this, &ElseGen, &D, RequiresOuterTask](
7894                              CodeGenFunction &CGF, PrePostActionTy &) {
7895     if (RequiresOuterTask) {
7896       CodeGenFunction::OMPTargetDataInfo InputInfo;
7897       CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
7898     } else {
7899       emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen);
7900     }
7901   };
7902
7903   // If we have a target function ID it means that we need to support
7904   // offloading, otherwise, just execute on the host. We need to execute on host
7905   // regardless of the conditional in the if clause if, e.g., the user do not
7906   // specify target triples.
7907   if (OutlinedFnID) {
7908     if (IfCond) {
7909       emitOMPIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
7910     } else {
7911       RegionCodeGenTy ThenRCG(TargetThenGen);
7912       ThenRCG(CGF);
7913     }
7914   } else {
7915     RegionCodeGenTy ElseRCG(TargetElseGen);
7916     ElseRCG(CGF);
7917   }
7918 }
7919
7920 void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
7921                                                     StringRef ParentName) {
7922   if (!S)
7923     return;
7924
7925   // Codegen OMP target directives that offload compute to the device.
7926   bool RequiresDeviceCodegen =
7927       isa<OMPExecutableDirective>(S) &&
7928       isOpenMPTargetExecutionDirective(
7929           cast<OMPExecutableDirective>(S)->getDirectiveKind());
7930
7931   if (RequiresDeviceCodegen) {
7932     const auto &E = *cast<OMPExecutableDirective>(S);
7933     unsigned DeviceID;
7934     unsigned FileID;
7935     unsigned Line;
7936     getTargetEntryUniqueInfo(CGM.getContext(), E.getLocStart(), DeviceID,
7937                              FileID, Line);
7938
7939     // Is this a target region that should not be emitted as an entry point? If
7940     // so just signal we are done with this target region.
7941     if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
7942                                                             ParentName, Line))
7943       return;
7944
7945     switch (E.getDirectiveKind()) {
7946     case OMPD_target:
7947       CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
7948                                                    cast<OMPTargetDirective>(E));
7949       break;
7950     case OMPD_target_parallel:
7951       CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
7952           CGM, ParentName, cast<OMPTargetParallelDirective>(E));
7953       break;
7954     case OMPD_target_teams:
7955       CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
7956           CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
7957       break;
7958     case OMPD_target_teams_distribute:
7959       CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
7960           CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
7961       break;
7962     case OMPD_target_teams_distribute_simd:
7963       CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
7964           CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
7965       break;
7966     case OMPD_target_parallel_for:
7967       CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
7968           CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
7969       break;
7970     case OMPD_target_parallel_for_simd:
7971       CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
7972           CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
7973       break;
7974     case OMPD_target_simd:
7975       CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
7976           CGM, ParentName, cast<OMPTargetSimdDirective>(E));
7977       break;
7978     case OMPD_target_teams_distribute_parallel_for:
7979       CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
7980           CGM, ParentName,
7981           cast<OMPTargetTeamsDistributeParallelForDirective>(E));
7982       break;
7983     case OMPD_target_teams_distribute_parallel_for_simd:
7984       CodeGenFunction::
7985           EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
7986               CGM, ParentName,
7987               cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
7988       break;
7989     case OMPD_parallel:
7990     case OMPD_for:
7991     case OMPD_parallel_for:
7992     case OMPD_parallel_sections:
7993     case OMPD_for_simd:
7994     case OMPD_parallel_for_simd:
7995     case OMPD_cancel:
7996     case OMPD_cancellation_point:
7997     case OMPD_ordered:
7998     case OMPD_threadprivate:
7999     case OMPD_task:
8000     case OMPD_simd:
8001     case OMPD_sections:
8002     case OMPD_section:
8003     case OMPD_single:
8004     case OMPD_master:
8005     case OMPD_critical:
8006     case OMPD_taskyield:
8007     case OMPD_barrier:
8008     case OMPD_taskwait:
8009     case OMPD_taskgroup:
8010     case OMPD_atomic:
8011     case OMPD_flush:
8012     case OMPD_teams:
8013     case OMPD_target_data:
8014     case OMPD_target_exit_data:
8015     case OMPD_target_enter_data:
8016     case OMPD_distribute:
8017     case OMPD_distribute_simd:
8018     case OMPD_distribute_parallel_for:
8019     case OMPD_distribute_parallel_for_simd:
8020     case OMPD_teams_distribute:
8021     case OMPD_teams_distribute_simd:
8022     case OMPD_teams_distribute_parallel_for:
8023     case OMPD_teams_distribute_parallel_for_simd:
8024     case OMPD_target_update:
8025     case OMPD_declare_simd:
8026     case OMPD_declare_target:
8027     case OMPD_end_declare_target:
8028     case OMPD_declare_reduction:
8029     case OMPD_taskloop:
8030     case OMPD_taskloop_simd:
8031     case OMPD_unknown:
8032       llvm_unreachable("Unknown target directive for OpenMP device codegen.");
8033     }
8034     return;
8035   }
8036
8037   if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
8038     if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
8039       return;
8040
8041     scanForTargetRegionsFunctions(
8042         E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
8043     return;
8044   }
8045
8046   // If this is a lambda function, look into its body.
8047   if (const auto *L = dyn_cast<LambdaExpr>(S))
8048     S = L->getBody();
8049
8050   // Keep looking for target regions recursively.
8051   for (const Stmt *II : S->children())
8052     scanForTargetRegionsFunctions(II, ParentName);
8053 }
8054
8055 bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
8056   const auto *FD = cast<FunctionDecl>(GD.getDecl());
8057
8058   // If emitting code for the host, we do not process FD here. Instead we do
8059   // the normal code generation.
8060   if (!CGM.getLangOpts().OpenMPIsDevice)
8061     return false;
8062
8063   // Try to detect target regions in the function.
8064   scanForTargetRegionsFunctions(FD->getBody(), CGM.getMangledName(GD));
8065
8066   // Do not to emit function if it is not marked as declare target.
8067   return !isDeclareTargetDeclaration(FD) &&
8068          AlreadyEmittedTargetFunctions.count(FD->getCanonicalDecl()) == 0;
8069 }
8070
8071 bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
8072   if (!CGM.getLangOpts().OpenMPIsDevice)
8073     return false;
8074
8075   // Check if there are Ctors/Dtors in this declaration and look for target
8076   // regions in it. We use the complete variant to produce the kernel name
8077   // mangling.
8078   QualType RDTy = cast<VarDecl>(GD.getDecl())->getType();
8079   if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
8080     for (const CXXConstructorDecl *Ctor : RD->ctors()) {
8081       StringRef ParentName =
8082           CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete));
8083       scanForTargetRegionsFunctions(Ctor->getBody(), ParentName);
8084     }
8085     if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
8086       StringRef ParentName =
8087           CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete));
8088       scanForTargetRegionsFunctions(Dtor->getBody(), ParentName);
8089     }
8090   }
8091
8092   // Do not to emit variable if it is not marked as declare target.
8093   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8094       isDeclareTargetDeclaration(cast<VarDecl>(GD.getDecl()));
8095   return !Res || *Res == OMPDeclareTargetDeclAttr::MT_Link;
8096 }
8097
8098 void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
8099                                                    llvm::Constant *Addr) {
8100   if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8101           isDeclareTargetDeclaration(VD)) {
8102     OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags;
8103     StringRef VarName;
8104     CharUnits VarSize;
8105     llvm::GlobalValue::LinkageTypes Linkage;
8106     switch (*Res) {
8107     case OMPDeclareTargetDeclAttr::MT_To:
8108       Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo;
8109       VarName = CGM.getMangledName(VD);
8110       VarSize = CGM.getContext().getTypeSizeInChars(VD->getType());
8111       Linkage = CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false);
8112       break;
8113     case OMPDeclareTargetDeclAttr::MT_Link:
8114       Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink;
8115       if (CGM.getLangOpts().OpenMPIsDevice) {
8116         VarName = Addr->getName();
8117         Addr = nullptr;
8118       } else {
8119         VarName = getAddrOfDeclareTargetLink(VD).getName();
8120         Addr =
8121             cast<llvm::Constant>(getAddrOfDeclareTargetLink(VD).getPointer());
8122       }
8123       VarSize = CGM.getPointerSize();
8124       Linkage = llvm::GlobalValue::WeakAnyLinkage;
8125       break;
8126     }
8127     OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
8128         VarName, Addr, VarSize, Flags, Linkage);
8129   }
8130 }
8131
8132 bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
8133   if (isa<FunctionDecl>(GD.getDecl()))
8134     return emitTargetFunctions(GD);
8135
8136   return emitTargetGlobalVariable(GD);
8137 }
8138
8139 CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
8140     CodeGenModule &CGM)
8141     : CGM(CGM) {
8142   if (CGM.getLangOpts().OpenMPIsDevice) {
8143     SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
8144     CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
8145   }
8146 }
8147
8148 CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
8149   if (CGM.getLangOpts().OpenMPIsDevice)
8150     CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
8151 }
8152
8153 bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
8154   if (!CGM.getLangOpts().OpenMPIsDevice || !ShouldMarkAsGlobal)
8155     return true;
8156
8157   const auto *D = cast<FunctionDecl>(GD.getDecl());
8158   const FunctionDecl *FD = D->getCanonicalDecl();
8159   // Do not to emit function if it is marked as declare target as it was already
8160   // emitted.
8161   if (isDeclareTargetDeclaration(D)) {
8162     if (D->hasBody() && AlreadyEmittedTargetFunctions.count(FD) == 0) {
8163       if (auto *F = dyn_cast_or_null<llvm::Function>(
8164               CGM.GetGlobalValue(CGM.getMangledName(GD))))
8165         return !F->isDeclaration();
8166       return false;
8167     }
8168     return true;
8169   }
8170
8171   return !AlreadyEmittedTargetFunctions.insert(FD).second;
8172 }
8173
8174 llvm::Function *CGOpenMPRuntime::emitRegistrationFunction() {
8175   // If we have offloading in the current module, we need to emit the entries
8176   // now and register the offloading descriptor.
8177   createOffloadEntriesAndInfoMetadata();
8178
8179   // Create and register the offloading binary descriptors. This is the main
8180   // entity that captures all the information about offloading in the current
8181   // compilation unit.
8182   return createOffloadingBinaryDescriptorRegistration();
8183 }
8184
8185 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
8186                                     const OMPExecutableDirective &D,
8187                                     SourceLocation Loc,
8188                                     llvm::Value *OutlinedFn,
8189                                     ArrayRef<llvm::Value *> CapturedVars) {
8190   if (!CGF.HaveInsertPoint())
8191     return;
8192
8193   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
8194   CodeGenFunction::RunCleanupsScope Scope(CGF);
8195
8196   // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
8197   llvm::Value *Args[] = {
8198       RTLoc,
8199       CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars
8200       CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())};
8201   llvm::SmallVector<llvm::Value *, 16> RealArgs;
8202   RealArgs.append(std::begin(Args), std::end(Args));
8203   RealArgs.append(CapturedVars.begin(), CapturedVars.end());
8204
8205   llvm::Value *RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_teams);
8206   CGF.EmitRuntimeCall(RTLFn, RealArgs);
8207 }
8208
8209 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
8210                                          const Expr *NumTeams,
8211                                          const Expr *ThreadLimit,
8212                                          SourceLocation Loc) {
8213   if (!CGF.HaveInsertPoint())
8214     return;
8215
8216   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
8217
8218   llvm::Value *NumTeamsVal =
8219       NumTeams
8220           ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams),
8221                                       CGF.CGM.Int32Ty, /* isSigned = */ true)
8222           : CGF.Builder.getInt32(0);
8223
8224   llvm::Value *ThreadLimitVal =
8225       ThreadLimit
8226           ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit),
8227                                       CGF.CGM.Int32Ty, /* isSigned = */ true)
8228           : CGF.Builder.getInt32(0);
8229
8230   // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
8231   llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
8232                                      ThreadLimitVal};
8233   CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams),
8234                       PushNumTeamsArgs);
8235 }
8236
8237 void CGOpenMPRuntime::emitTargetDataCalls(
8238     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
8239     const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
8240   if (!CGF.HaveInsertPoint())
8241     return;
8242
8243   // Action used to replace the default codegen action and turn privatization
8244   // off.
8245   PrePostActionTy NoPrivAction;
8246
8247   // Generate the code for the opening of the data environment. Capture all the
8248   // arguments of the runtime call by reference because they are used in the
8249   // closing of the region.
8250   auto &&BeginThenGen = [this, &D, Device, &Info,
8251                          &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
8252     // Fill up the arrays with all the mapped variables.
8253     MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
8254     MappableExprsHandler::MapValuesArrayTy Pointers;
8255     MappableExprsHandler::MapValuesArrayTy Sizes;
8256     MappableExprsHandler::MapFlagsArrayTy MapTypes;
8257
8258     // Get map clause information.
8259     MappableExprsHandler MCHandler(D, CGF);
8260     MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
8261
8262     // Fill up the arrays and create the arguments.
8263     emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
8264
8265     llvm::Value *BasePointersArrayArg = nullptr;
8266     llvm::Value *PointersArrayArg = nullptr;
8267     llvm::Value *SizesArrayArg = nullptr;
8268     llvm::Value *MapTypesArrayArg = nullptr;
8269     emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
8270                                  SizesArrayArg, MapTypesArrayArg, Info);
8271
8272     // Emit device ID if any.
8273     llvm::Value *DeviceID = nullptr;
8274     if (Device) {
8275       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8276                                            CGF.Int64Ty, /*isSigned=*/true);
8277     } else {
8278       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8279     }
8280
8281     // Emit the number of elements in the offloading arrays.
8282     llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
8283
8284     llvm::Value *OffloadingArgs[] = {
8285         DeviceID,         PointerNum,    BasePointersArrayArg,
8286         PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
8287     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin),
8288                         OffloadingArgs);
8289
8290     // If device pointer privatization is required, emit the body of the region
8291     // here. It will have to be duplicated: with and without privatization.
8292     if (!Info.CaptureDeviceAddrMap.empty())
8293       CodeGen(CGF);
8294   };
8295
8296   // Generate code for the closing of the data region.
8297   auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF,
8298                                             PrePostActionTy &) {
8299     assert(Info.isValid() && "Invalid data environment closing arguments.");
8300
8301     llvm::Value *BasePointersArrayArg = nullptr;
8302     llvm::Value *PointersArrayArg = nullptr;
8303     llvm::Value *SizesArrayArg = nullptr;
8304     llvm::Value *MapTypesArrayArg = nullptr;
8305     emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg,
8306                                  SizesArrayArg, MapTypesArrayArg, Info);
8307
8308     // Emit device ID if any.
8309     llvm::Value *DeviceID = nullptr;
8310     if (Device) {
8311       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8312                                            CGF.Int64Ty, /*isSigned=*/true);
8313     } else {
8314       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8315     }
8316
8317     // Emit the number of elements in the offloading arrays.
8318     llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
8319
8320     llvm::Value *OffloadingArgs[] = {
8321         DeviceID,         PointerNum,    BasePointersArrayArg,
8322         PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
8323     CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end),
8324                         OffloadingArgs);
8325   };
8326
8327   // If we need device pointer privatization, we need to emit the body of the
8328   // region with no privatization in the 'else' branch of the conditional.
8329   // Otherwise, we don't have to do anything.
8330   auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF,
8331                                                          PrePostActionTy &) {
8332     if (!Info.CaptureDeviceAddrMap.empty()) {
8333       CodeGen.setAction(NoPrivAction);
8334       CodeGen(CGF);
8335     }
8336   };
8337
8338   // We don't have to do anything to close the region if the if clause evaluates
8339   // to false.
8340   auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {};
8341
8342   if (IfCond) {
8343     emitOMPIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
8344   } else {
8345     RegionCodeGenTy RCG(BeginThenGen);
8346     RCG(CGF);
8347   }
8348
8349   // If we don't require privatization of device pointers, we emit the body in
8350   // between the runtime calls. This avoids duplicating the body code.
8351   if (Info.CaptureDeviceAddrMap.empty()) {
8352     CodeGen.setAction(NoPrivAction);
8353     CodeGen(CGF);
8354   }
8355
8356   if (IfCond) {
8357     emitOMPIfClause(CGF, IfCond, EndThenGen, EndElseGen);
8358   } else {
8359     RegionCodeGenTy RCG(EndThenGen);
8360     RCG(CGF);
8361   }
8362 }
8363
8364 void CGOpenMPRuntime::emitTargetDataStandAloneCall(
8365     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
8366     const Expr *Device) {
8367   if (!CGF.HaveInsertPoint())
8368     return;
8369
8370   assert((isa<OMPTargetEnterDataDirective>(D) ||
8371           isa<OMPTargetExitDataDirective>(D) ||
8372           isa<OMPTargetUpdateDirective>(D)) &&
8373          "Expecting either target enter, exit data, or update directives.");
8374
8375   CodeGenFunction::OMPTargetDataInfo InputInfo;
8376   llvm::Value *MapTypesArray = nullptr;
8377   // Generate the code for the opening of the data environment.
8378   auto &&ThenGen = [this, &D, Device, &InputInfo,
8379                     &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) {
8380     // Emit device ID if any.
8381     llvm::Value *DeviceID = nullptr;
8382     if (Device) {
8383       DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
8384                                            CGF.Int64Ty, /*isSigned=*/true);
8385     } else {
8386       DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
8387     }
8388
8389     // Emit the number of elements in the offloading arrays.
8390     llvm::Constant *PointerNum =
8391         CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
8392
8393     llvm::Value *OffloadingArgs[] = {DeviceID,
8394                                      PointerNum,
8395                                      InputInfo.BasePointersArray.getPointer(),
8396                                      InputInfo.PointersArray.getPointer(),
8397                                      InputInfo.SizesArray.getPointer(),
8398                                      MapTypesArray};
8399
8400     // Select the right runtime function call for each expected standalone
8401     // directive.
8402     const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
8403     OpenMPRTLFunction RTLFn;
8404     switch (D.getDirectiveKind()) {
8405     case OMPD_target_enter_data:
8406       RTLFn = HasNowait ? OMPRTL__tgt_target_data_begin_nowait
8407                         : OMPRTL__tgt_target_data_begin;
8408       break;
8409     case OMPD_target_exit_data:
8410       RTLFn = HasNowait ? OMPRTL__tgt_target_data_end_nowait
8411                         : OMPRTL__tgt_target_data_end;
8412       break;
8413     case OMPD_target_update:
8414       RTLFn = HasNowait ? OMPRTL__tgt_target_data_update_nowait
8415                         : OMPRTL__tgt_target_data_update;
8416       break;
8417     case OMPD_parallel:
8418     case OMPD_for:
8419     case OMPD_parallel_for:
8420     case OMPD_parallel_sections:
8421     case OMPD_for_simd:
8422     case OMPD_parallel_for_simd:
8423     case OMPD_cancel:
8424     case OMPD_cancellation_point:
8425     case OMPD_ordered:
8426     case OMPD_threadprivate:
8427     case OMPD_task:
8428     case OMPD_simd:
8429     case OMPD_sections:
8430     case OMPD_section:
8431     case OMPD_single:
8432     case OMPD_master:
8433     case OMPD_critical:
8434     case OMPD_taskyield:
8435     case OMPD_barrier:
8436     case OMPD_taskwait:
8437     case OMPD_taskgroup:
8438     case OMPD_atomic:
8439     case OMPD_flush:
8440     case OMPD_teams:
8441     case OMPD_target_data:
8442     case OMPD_distribute:
8443     case OMPD_distribute_simd:
8444     case OMPD_distribute_parallel_for:
8445     case OMPD_distribute_parallel_for_simd:
8446     case OMPD_teams_distribute:
8447     case OMPD_teams_distribute_simd:
8448     case OMPD_teams_distribute_parallel_for:
8449     case OMPD_teams_distribute_parallel_for_simd:
8450     case OMPD_declare_simd:
8451     case OMPD_declare_target:
8452     case OMPD_end_declare_target:
8453     case OMPD_declare_reduction:
8454     case OMPD_taskloop:
8455     case OMPD_taskloop_simd:
8456     case OMPD_target:
8457     case OMPD_target_simd:
8458     case OMPD_target_teams_distribute:
8459     case OMPD_target_teams_distribute_simd:
8460     case OMPD_target_teams_distribute_parallel_for:
8461     case OMPD_target_teams_distribute_parallel_for_simd:
8462     case OMPD_target_teams:
8463     case OMPD_target_parallel:
8464     case OMPD_target_parallel_for:
8465     case OMPD_target_parallel_for_simd:
8466     case OMPD_unknown:
8467       llvm_unreachable("Unexpected standalone target data directive.");
8468       break;
8469     }
8470     CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs);
8471   };
8472
8473   auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray](
8474                              CodeGenFunction &CGF, PrePostActionTy &) {
8475     // Fill up the arrays with all the mapped variables.
8476     MappableExprsHandler::MapBaseValuesArrayTy BasePointers;
8477     MappableExprsHandler::MapValuesArrayTy Pointers;
8478     MappableExprsHandler::MapValuesArrayTy Sizes;
8479     MappableExprsHandler::MapFlagsArrayTy MapTypes;
8480
8481     // Get map clause information.
8482     MappableExprsHandler MEHandler(D, CGF);
8483     MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
8484
8485     TargetDataInfo Info;
8486     // Fill up the arrays and create the arguments.
8487     emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info);
8488     emitOffloadingArraysArgument(CGF, Info.BasePointersArray,
8489                                  Info.PointersArray, Info.SizesArray,
8490                                  Info.MapTypesArray, Info);
8491     InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
8492     InputInfo.BasePointersArray =
8493         Address(Info.BasePointersArray, CGM.getPointerAlign());
8494     InputInfo.PointersArray =
8495         Address(Info.PointersArray, CGM.getPointerAlign());
8496     InputInfo.SizesArray =
8497         Address(Info.SizesArray, CGM.getPointerAlign());
8498     MapTypesArray = Info.MapTypesArray;
8499     if (D.hasClausesOfKind<OMPDependClause>())
8500       CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
8501     else
8502       emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
8503   };
8504
8505   if (IfCond) {
8506     emitOMPIfClause(CGF, IfCond, TargetThenGen,
8507                     [](CodeGenFunction &CGF, PrePostActionTy &) {});
8508   } else {
8509     RegionCodeGenTy ThenRCG(TargetThenGen);
8510     ThenRCG(CGF);
8511   }
8512 }
8513
8514 namespace {
8515   /// Kind of parameter in a function with 'declare simd' directive.
8516   enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector };
8517   /// Attribute set of the parameter.
8518   struct ParamAttrTy {
8519     ParamKindTy Kind = Vector;
8520     llvm::APSInt StrideOrArg;
8521     llvm::APSInt Alignment;
8522   };
8523 } // namespace
8524
8525 static unsigned evaluateCDTSize(const FunctionDecl *FD,
8526                                 ArrayRef<ParamAttrTy> ParamAttrs) {
8527   // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
8528   // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
8529   // of that clause. The VLEN value must be power of 2.
8530   // In other case the notion of the function`s "characteristic data type" (CDT)
8531   // is used to compute the vector length.
8532   // CDT is defined in the following order:
8533   //   a) For non-void function, the CDT is the return type.
8534   //   b) If the function has any non-uniform, non-linear parameters, then the
8535   //   CDT is the type of the first such parameter.
8536   //   c) If the CDT determined by a) or b) above is struct, union, or class
8537   //   type which is pass-by-value (except for the type that maps to the
8538   //   built-in complex data type), the characteristic data type is int.
8539   //   d) If none of the above three cases is applicable, the CDT is int.
8540   // The VLEN is then determined based on the CDT and the size of vector
8541   // register of that ISA for which current vector version is generated. The
8542   // VLEN is computed using the formula below:
8543   //   VLEN  = sizeof(vector_register) / sizeof(CDT),
8544   // where vector register size specified in section 3.2.1 Registers and the
8545   // Stack Frame of original AMD64 ABI document.
8546   QualType RetType = FD->getReturnType();
8547   if (RetType.isNull())
8548     return 0;
8549   ASTContext &C = FD->getASTContext();
8550   QualType CDT;
8551   if (!RetType.isNull() && !RetType->isVoidType()) {
8552     CDT = RetType;
8553   } else {
8554     unsigned Offset = 0;
8555     if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
8556       if (ParamAttrs[Offset].Kind == Vector)
8557         CDT = C.getPointerType(C.getRecordType(MD->getParent()));
8558       ++Offset;
8559     }
8560     if (CDT.isNull()) {
8561       for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
8562         if (ParamAttrs[I + Offset].Kind == Vector) {
8563           CDT = FD->getParamDecl(I)->getType();
8564           break;
8565         }
8566       }
8567     }
8568   }
8569   if (CDT.isNull())
8570     CDT = C.IntTy;
8571   CDT = CDT->getCanonicalTypeUnqualified();
8572   if (CDT->isRecordType() || CDT->isUnionType())
8573     CDT = C.IntTy;
8574   return C.getTypeSize(CDT);
8575 }
8576
8577 static void
8578 emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn,
8579                            const llvm::APSInt &VLENVal,
8580                            ArrayRef<ParamAttrTy> ParamAttrs,
8581                            OMPDeclareSimdDeclAttr::BranchStateTy State) {
8582   struct ISADataTy {
8583     char ISA;
8584     unsigned VecRegSize;
8585   };
8586   ISADataTy ISAData[] = {
8587       {
8588           'b', 128
8589       }, // SSE
8590       {
8591           'c', 256
8592       }, // AVX
8593       {
8594           'd', 256
8595       }, // AVX2
8596       {
8597           'e', 512
8598       }, // AVX512
8599   };
8600   llvm::SmallVector<char, 2> Masked;
8601   switch (State) {
8602   case OMPDeclareSimdDeclAttr::BS_Undefined:
8603     Masked.push_back('N');
8604     Masked.push_back('M');
8605     break;
8606   case OMPDeclareSimdDeclAttr::BS_Notinbranch:
8607     Masked.push_back('N');
8608     break;
8609   case OMPDeclareSimdDeclAttr::BS_Inbranch:
8610     Masked.push_back('M');
8611     break;
8612   }
8613   for (char Mask : Masked) {
8614     for (const ISADataTy &Data : ISAData) {
8615       SmallString<256> Buffer;
8616       llvm::raw_svector_ostream Out(Buffer);
8617       Out << "_ZGV" << Data.ISA << Mask;
8618       if (!VLENVal) {
8619         Out << llvm::APSInt::getUnsigned(Data.VecRegSize /
8620                                          evaluateCDTSize(FD, ParamAttrs));
8621       } else {
8622         Out << VLENVal;
8623       }
8624       for (const ParamAttrTy &ParamAttr : ParamAttrs) {
8625         switch (ParamAttr.Kind){
8626         case LinearWithVarStride:
8627           Out << 's' << ParamAttr.StrideOrArg;
8628           break;
8629         case Linear:
8630           Out << 'l';
8631           if (!!ParamAttr.StrideOrArg)
8632             Out << ParamAttr.StrideOrArg;
8633           break;
8634         case Uniform:
8635           Out << 'u';
8636           break;
8637         case Vector:
8638           Out << 'v';
8639           break;
8640         }
8641         if (!!ParamAttr.Alignment)
8642           Out << 'a' << ParamAttr.Alignment;
8643       }
8644       Out << '_' << Fn->getName();
8645       Fn->addFnAttr(Out.str());
8646     }
8647   }
8648 }
8649
8650 void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
8651                                               llvm::Function *Fn) {
8652   ASTContext &C = CGM.getContext();
8653   FD = FD->getMostRecentDecl();
8654   // Map params to their positions in function decl.
8655   llvm::DenseMap<const Decl *, unsigned> ParamPositions;
8656   if (isa<CXXMethodDecl>(FD))
8657     ParamPositions.try_emplace(FD, 0);
8658   unsigned ParamPos = ParamPositions.size();
8659   for (const ParmVarDecl *P : FD->parameters()) {
8660     ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos);
8661     ++ParamPos;
8662   }
8663   while (FD) {
8664     for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
8665       llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size());
8666       // Mark uniform parameters.
8667       for (const Expr *E : Attr->uniforms()) {
8668         E = E->IgnoreParenImpCasts();
8669         unsigned Pos;
8670         if (isa<CXXThisExpr>(E)) {
8671           Pos = ParamPositions[FD];
8672         } else {
8673           const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
8674                                 ->getCanonicalDecl();
8675           Pos = ParamPositions[PVD];
8676         }
8677         ParamAttrs[Pos].Kind = Uniform;
8678       }
8679       // Get alignment info.
8680       auto NI = Attr->alignments_begin();
8681       for (const Expr *E : Attr->aligneds()) {
8682         E = E->IgnoreParenImpCasts();
8683         unsigned Pos;
8684         QualType ParmTy;
8685         if (isa<CXXThisExpr>(E)) {
8686           Pos = ParamPositions[FD];
8687           ParmTy = E->getType();
8688         } else {
8689           const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
8690                                 ->getCanonicalDecl();
8691           Pos = ParamPositions[PVD];
8692           ParmTy = PVD->getType();
8693         }
8694         ParamAttrs[Pos].Alignment =
8695             (*NI)
8696                 ? (*NI)->EvaluateKnownConstInt(C)
8697                 : llvm::APSInt::getUnsigned(
8698                       C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy))
8699                           .getQuantity());
8700         ++NI;
8701       }
8702       // Mark linear parameters.
8703       auto SI = Attr->steps_begin();
8704       auto MI = Attr->modifiers_begin();
8705       for (const Expr *E : Attr->linears()) {
8706         E = E->IgnoreParenImpCasts();
8707         unsigned Pos;
8708         if (isa<CXXThisExpr>(E)) {
8709           Pos = ParamPositions[FD];
8710         } else {
8711           const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
8712                                 ->getCanonicalDecl();
8713           Pos = ParamPositions[PVD];
8714         }
8715         ParamAttrTy &ParamAttr = ParamAttrs[Pos];
8716         ParamAttr.Kind = Linear;
8717         if (*SI) {
8718           if (!(*SI)->EvaluateAsInt(ParamAttr.StrideOrArg, C,
8719                                     Expr::SE_AllowSideEffects)) {
8720             if (const auto *DRE =
8721                     cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
8722               if (const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
8723                 ParamAttr.Kind = LinearWithVarStride;
8724                 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
8725                     ParamPositions[StridePVD->getCanonicalDecl()]);
8726               }
8727             }
8728           }
8729         }
8730         ++SI;
8731         ++MI;
8732       }
8733       llvm::APSInt VLENVal;
8734       if (const Expr *VLEN = Attr->getSimdlen())
8735         VLENVal = VLEN->EvaluateKnownConstInt(C);
8736       OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState();
8737       if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
8738           CGM.getTriple().getArch() == llvm::Triple::x86_64)
8739         emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State);
8740     }
8741     FD = FD->getPreviousDecl();
8742   }
8743 }
8744
8745 namespace {
8746 /// Cleanup action for doacross support.
8747 class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
8748 public:
8749   static const int DoacrossFinArgs = 2;
8750
8751 private:
8752   llvm::Value *RTLFn;
8753   llvm::Value *Args[DoacrossFinArgs];
8754
8755 public:
8756   DoacrossCleanupTy(llvm::Value *RTLFn, ArrayRef<llvm::Value *> CallArgs)
8757       : RTLFn(RTLFn) {
8758     assert(CallArgs.size() == DoacrossFinArgs);
8759     std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
8760   }
8761   void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
8762     if (!CGF.HaveInsertPoint())
8763       return;
8764     CGF.EmitRuntimeCall(RTLFn, Args);
8765   }
8766 };
8767 } // namespace
8768
8769 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
8770                                        const OMPLoopDirective &D) {
8771   if (!CGF.HaveInsertPoint())
8772     return;
8773
8774   ASTContext &C = CGM.getContext();
8775   QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
8776   RecordDecl *RD;
8777   if (KmpDimTy.isNull()) {
8778     // Build struct kmp_dim {  // loop bounds info casted to kmp_int64
8779     //  kmp_int64 lo; // lower
8780     //  kmp_int64 up; // upper
8781     //  kmp_int64 st; // stride
8782     // };
8783     RD = C.buildImplicitRecord("kmp_dim");
8784     RD->startDefinition();
8785     addFieldToRecordDecl(C, RD, Int64Ty);
8786     addFieldToRecordDecl(C, RD, Int64Ty);
8787     addFieldToRecordDecl(C, RD, Int64Ty);
8788     RD->completeDefinition();
8789     KmpDimTy = C.getRecordType(RD);
8790   } else {
8791     RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl());
8792   }
8793
8794   Address DimsAddr = CGF.CreateMemTemp(KmpDimTy, "dims");
8795   CGF.EmitNullInitialization(DimsAddr, KmpDimTy);
8796   enum { LowerFD = 0, UpperFD, StrideFD };
8797   // Fill dims with data.
8798   LValue DimsLVal = CGF.MakeAddrLValue(DimsAddr, KmpDimTy);
8799   // dims.upper = num_iterations;
8800   LValue UpperLVal =
8801       CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), UpperFD));
8802   llvm::Value *NumIterVal = CGF.EmitScalarConversion(
8803       CGF.EmitScalarExpr(D.getNumIterations()), D.getNumIterations()->getType(),
8804       Int64Ty, D.getNumIterations()->getExprLoc());
8805   CGF.EmitStoreOfScalar(NumIterVal, UpperLVal);
8806   // dims.stride = 1;
8807   LValue StrideLVal =
8808       CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), StrideFD));
8809   CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1),
8810                         StrideLVal);
8811
8812   // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
8813   // kmp_int32 num_dims, struct kmp_dim * dims);
8814   llvm::Value *Args[] = {emitUpdateLocation(CGF, D.getLocStart()),
8815                          getThreadID(CGF, D.getLocStart()),
8816                          llvm::ConstantInt::getSigned(CGM.Int32Ty, 1),
8817                          CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8818                              DimsAddr.getPointer(), CGM.VoidPtrTy)};
8819
8820   llvm::Value *RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_init);
8821   CGF.EmitRuntimeCall(RTLFn, Args);
8822   llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
8823       emitUpdateLocation(CGF, D.getLocEnd()), getThreadID(CGF, D.getLocEnd())};
8824   llvm::Value *FiniRTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_fini);
8825   CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn,
8826                                              llvm::makeArrayRef(FiniArgs));
8827 }
8828
8829 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
8830                                           const OMPDependClause *C) {
8831   QualType Int64Ty =
8832       CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
8833   const Expr *CounterVal = C->getCounterValue();
8834   assert(CounterVal);
8835   llvm::Value *CntVal = CGF.EmitScalarConversion(CGF.EmitScalarExpr(CounterVal),
8836                                                  CounterVal->getType(), Int64Ty,
8837                                                  CounterVal->getExprLoc());
8838   Address CntAddr = CGF.CreateMemTemp(Int64Ty, ".cnt.addr");
8839   CGF.EmitStoreOfScalar(CntVal, CntAddr, /*Volatile=*/false, Int64Ty);
8840   llvm::Value *Args[] = {emitUpdateLocation(CGF, C->getLocStart()),
8841                          getThreadID(CGF, C->getLocStart()),
8842                          CntAddr.getPointer()};
8843   llvm::Value *RTLFn;
8844   if (C->getDependencyKind() == OMPC_DEPEND_source) {
8845     RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_post);
8846   } else {
8847     assert(C->getDependencyKind() == OMPC_DEPEND_sink);
8848     RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_wait);
8849   }
8850   CGF.EmitRuntimeCall(RTLFn, Args);
8851 }
8852
8853 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
8854                                llvm::Value *Callee,
8855                                ArrayRef<llvm::Value *> Args) const {
8856   assert(Loc.isValid() && "Outlined function call location must be valid.");
8857   auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
8858
8859   if (auto *Fn = dyn_cast<llvm::Function>(Callee)) {
8860     if (Fn->doesNotThrow()) {
8861       CGF.EmitNounwindRuntimeCall(Fn, Args);
8862       return;
8863     }
8864   }
8865   CGF.EmitRuntimeCall(Callee, Args);
8866 }
8867
8868 void CGOpenMPRuntime::emitOutlinedFunctionCall(
8869     CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
8870     ArrayRef<llvm::Value *> Args) const {
8871   emitCall(CGF, Loc, OutlinedFn, Args);
8872 }
8873
8874 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
8875                                              const VarDecl *NativeParam,
8876                                              const VarDecl *TargetParam) const {
8877   return CGF.GetAddrOfLocalVar(NativeParam);
8878 }
8879
8880 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
8881                                                    const VarDecl *VD) {
8882   return Address::invalid();
8883 }
8884
8885 llvm::Value *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
8886     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
8887     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
8888   llvm_unreachable("Not supported in SIMD-only mode");
8889 }
8890
8891 llvm::Value *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
8892     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
8893     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
8894   llvm_unreachable("Not supported in SIMD-only mode");
8895 }
8896
8897 llvm::Value *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
8898     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
8899     const VarDecl *PartIDVar, const VarDecl *TaskTVar,
8900     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
8901     bool Tied, unsigned &NumberOfParts) {
8902   llvm_unreachable("Not supported in SIMD-only mode");
8903 }
8904
8905 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF,
8906                                            SourceLocation Loc,
8907                                            llvm::Value *OutlinedFn,
8908                                            ArrayRef<llvm::Value *> CapturedVars,
8909                                            const Expr *IfCond) {
8910   llvm_unreachable("Not supported in SIMD-only mode");
8911 }
8912
8913 void CGOpenMPSIMDRuntime::emitCriticalRegion(
8914     CodeGenFunction &CGF, StringRef CriticalName,
8915     const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
8916     const Expr *Hint) {
8917   llvm_unreachable("Not supported in SIMD-only mode");
8918 }
8919
8920 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
8921                                            const RegionCodeGenTy &MasterOpGen,
8922                                            SourceLocation Loc) {
8923   llvm_unreachable("Not supported in SIMD-only mode");
8924 }
8925
8926 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
8927                                             SourceLocation Loc) {
8928   llvm_unreachable("Not supported in SIMD-only mode");
8929 }
8930
8931 void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
8932     CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
8933     SourceLocation Loc) {
8934   llvm_unreachable("Not supported in SIMD-only mode");
8935 }
8936
8937 void CGOpenMPSIMDRuntime::emitSingleRegion(
8938     CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
8939     SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
8940     ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
8941     ArrayRef<const Expr *> AssignmentOps) {
8942   llvm_unreachable("Not supported in SIMD-only mode");
8943 }
8944
8945 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
8946                                             const RegionCodeGenTy &OrderedOpGen,
8947                                             SourceLocation Loc,
8948                                             bool IsThreads) {
8949   llvm_unreachable("Not supported in SIMD-only mode");
8950 }
8951
8952 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
8953                                           SourceLocation Loc,
8954                                           OpenMPDirectiveKind Kind,
8955                                           bool EmitChecks,
8956                                           bool ForceSimpleCall) {
8957   llvm_unreachable("Not supported in SIMD-only mode");
8958 }
8959
8960 void CGOpenMPSIMDRuntime::emitForDispatchInit(
8961     CodeGenFunction &CGF, SourceLocation Loc,
8962     const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
8963     bool Ordered, const DispatchRTInput &DispatchValues) {
8964   llvm_unreachable("Not supported in SIMD-only mode");
8965 }
8966
8967 void CGOpenMPSIMDRuntime::emitForStaticInit(
8968     CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
8969     const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
8970   llvm_unreachable("Not supported in SIMD-only mode");
8971 }
8972
8973 void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
8974     CodeGenFunction &CGF, SourceLocation Loc,
8975     OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
8976   llvm_unreachable("Not supported in SIMD-only mode");
8977 }
8978
8979 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
8980                                                      SourceLocation Loc,
8981                                                      unsigned IVSize,
8982                                                      bool IVSigned) {
8983   llvm_unreachable("Not supported in SIMD-only mode");
8984 }
8985
8986 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
8987                                               SourceLocation Loc,
8988                                               OpenMPDirectiveKind DKind) {
8989   llvm_unreachable("Not supported in SIMD-only mode");
8990 }
8991
8992 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
8993                                               SourceLocation Loc,
8994                                               unsigned IVSize, bool IVSigned,
8995                                               Address IL, Address LB,
8996                                               Address UB, Address ST) {
8997   llvm_unreachable("Not supported in SIMD-only mode");
8998 }
8999
9000 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF,
9001                                                llvm::Value *NumThreads,
9002                                                SourceLocation Loc) {
9003   llvm_unreachable("Not supported in SIMD-only mode");
9004 }
9005
9006 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
9007                                              OpenMPProcBindClauseKind ProcBind,
9008                                              SourceLocation Loc) {
9009   llvm_unreachable("Not supported in SIMD-only mode");
9010 }
9011
9012 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
9013                                                     const VarDecl *VD,
9014                                                     Address VDAddr,
9015                                                     SourceLocation Loc) {
9016   llvm_unreachable("Not supported in SIMD-only mode");
9017 }
9018
9019 llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
9020     const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
9021     CodeGenFunction *CGF) {
9022   llvm_unreachable("Not supported in SIMD-only mode");
9023 }
9024
9025 Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
9026     CodeGenFunction &CGF, QualType VarType, StringRef Name) {
9027   llvm_unreachable("Not supported in SIMD-only mode");
9028 }
9029
9030 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
9031                                     ArrayRef<const Expr *> Vars,
9032                                     SourceLocation Loc) {
9033   llvm_unreachable("Not supported in SIMD-only mode");
9034 }
9035
9036 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
9037                                        const OMPExecutableDirective &D,
9038                                        llvm::Value *TaskFunction,
9039                                        QualType SharedsTy, Address Shareds,
9040                                        const Expr *IfCond,
9041                                        const OMPTaskDataTy &Data) {
9042   llvm_unreachable("Not supported in SIMD-only mode");
9043 }
9044
9045 void CGOpenMPSIMDRuntime::emitTaskLoopCall(
9046     CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
9047     llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds,
9048     const Expr *IfCond, const OMPTaskDataTy &Data) {
9049   llvm_unreachable("Not supported in SIMD-only mode");
9050 }
9051
9052 void CGOpenMPSIMDRuntime::emitReduction(
9053     CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
9054     ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
9055     ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
9056   assert(Options.SimpleReduction && "Only simple reduction is expected.");
9057   CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs,
9058                                  ReductionOps, Options);
9059 }
9060
9061 llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
9062     CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
9063     ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
9064   llvm_unreachable("Not supported in SIMD-only mode");
9065 }
9066
9067 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
9068                                                   SourceLocation Loc,
9069                                                   ReductionCodeGen &RCG,
9070                                                   unsigned N) {
9071   llvm_unreachable("Not supported in SIMD-only mode");
9072 }
9073
9074 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
9075                                                   SourceLocation Loc,
9076                                                   llvm::Value *ReductionsPtr,
9077                                                   LValue SharedLVal) {
9078   llvm_unreachable("Not supported in SIMD-only mode");
9079 }
9080
9081 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
9082                                            SourceLocation Loc) {
9083   llvm_unreachable("Not supported in SIMD-only mode");
9084 }
9085
9086 void CGOpenMPSIMDRuntime::emitCancellationPointCall(
9087     CodeGenFunction &CGF, SourceLocation Loc,
9088     OpenMPDirectiveKind CancelRegion) {
9089   llvm_unreachable("Not supported in SIMD-only mode");
9090 }
9091
9092 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
9093                                          SourceLocation Loc, const Expr *IfCond,
9094                                          OpenMPDirectiveKind CancelRegion) {
9095   llvm_unreachable("Not supported in SIMD-only mode");
9096 }
9097
9098 void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
9099     const OMPExecutableDirective &D, StringRef ParentName,
9100     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
9101     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
9102   llvm_unreachable("Not supported in SIMD-only mode");
9103 }
9104
9105 void CGOpenMPSIMDRuntime::emitTargetCall(CodeGenFunction &CGF,
9106                                          const OMPExecutableDirective &D,
9107                                          llvm::Value *OutlinedFn,
9108                                          llvm::Value *OutlinedFnID,
9109                                          const Expr *IfCond, const Expr *Device) {
9110   llvm_unreachable("Not supported in SIMD-only mode");
9111 }
9112
9113 bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
9114   llvm_unreachable("Not supported in SIMD-only mode");
9115 }
9116
9117 bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
9118   llvm_unreachable("Not supported in SIMD-only mode");
9119 }
9120
9121 bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
9122   return false;
9123 }
9124
9125 llvm::Function *CGOpenMPSIMDRuntime::emitRegistrationFunction() {
9126   return nullptr;
9127 }
9128
9129 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
9130                                         const OMPExecutableDirective &D,
9131                                         SourceLocation Loc,
9132                                         llvm::Value *OutlinedFn,
9133                                         ArrayRef<llvm::Value *> CapturedVars) {
9134   llvm_unreachable("Not supported in SIMD-only mode");
9135 }
9136
9137 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
9138                                              const Expr *NumTeams,
9139                                              const Expr *ThreadLimit,
9140                                              SourceLocation Loc) {
9141   llvm_unreachable("Not supported in SIMD-only mode");
9142 }
9143
9144 void CGOpenMPSIMDRuntime::emitTargetDataCalls(
9145     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
9146     const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) {
9147   llvm_unreachable("Not supported in SIMD-only mode");
9148 }
9149
9150 void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
9151     CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
9152     const Expr *Device) {
9153   llvm_unreachable("Not supported in SIMD-only mode");
9154 }
9155
9156 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
9157                                            const OMPLoopDirective &D) {
9158   llvm_unreachable("Not supported in SIMD-only mode");
9159 }
9160
9161 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
9162                                               const OMPDependClause *C) {
9163   llvm_unreachable("Not supported in SIMD-only mode");
9164 }
9165
9166 const VarDecl *
9167 CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
9168                                         const VarDecl *NativeParam) const {
9169   llvm_unreachable("Not supported in SIMD-only mode");
9170 }
9171
9172 Address
9173 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
9174                                          const VarDecl *NativeParam,
9175                                          const VarDecl *TargetParam) const {
9176   llvm_unreachable("Not supported in SIMD-only mode");
9177 }
9178