contrib/openmp/runtime/src/kmp_gsupport.cpp

   1 /*
   2  * kmp_gsupport.cpp
   3  */
   4
   5 //===----------------------------------------------------------------------===//
   6 //
   7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   8 // See https://llvm.org/LICENSE.txt for license information.
   9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "kmp.h"
  14 #include "kmp_atomic.h"
  15
  16 #if OMPT_SUPPORT
  17 #include "ompt-specific.h"
  18 #endif
  19
  20 #ifdef __cplusplus
  21 extern "C" {
  22 #endif // __cplusplus
  23
  24 #define MKLOC(loc, routine)                                                    \
  25   static ident_t(loc) = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"};
  26
  27 #include "kmp_ftn_os.h"
  28
  29 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER)(void) {
  30   int gtid = __kmp_entry_gtid();
  31   MKLOC(loc, "GOMP_barrier");
  32   KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid));
  33 #if OMPT_SUPPORT && OMPT_OPTIONAL
  34   ompt_frame_t *ompt_frame;
  35   if (ompt_enabled.enabled) {
  36     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
  37     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
  38     OMPT_STORE_RETURN_ADDRESS(gtid);
  39   }
  40 #endif
  41   __kmpc_barrier(&loc, gtid);
  42 #if OMPT_SUPPORT && OMPT_OPTIONAL
  43   if (ompt_enabled.enabled) {
  44     ompt_frame->enter_frame = ompt_data_none;
  45   }
  46 #endif
  47 }
  48
  49 // Mutual exclusion
  50
  51 // The symbol that icc/ifort generates for unnamed for unnamed critical sections
  52 // - .gomp_critical_user_ - is defined using .comm in any objects reference it.
  53 // We can't reference it directly here in C code, as the symbol contains a ".".
  54 //
  55 // The RTL contains an assembly language definition of .gomp_critical_user_
  56 // with another symbol __kmp_unnamed_critical_addr initialized with it's
  57 // address.
  58 extern kmp_critical_name *__kmp_unnamed_critical_addr;
  59
  60 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_START)(void) {
  61   int gtid = __kmp_entry_gtid();
  62   MKLOC(loc, "GOMP_critical_start");
  63   KA_TRACE(20, ("GOMP_critical_start: T#%d\n", gtid));
  64 #if OMPT_SUPPORT && OMPT_OPTIONAL
  65   OMPT_STORE_RETURN_ADDRESS(gtid);
  66 #endif
  67   __kmpc_critical(&loc, gtid, __kmp_unnamed_critical_addr);
  68 }
  69
  70 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_END)(void) {
  71   int gtid = __kmp_get_gtid();
  72   MKLOC(loc, "GOMP_critical_end");
  73   KA_TRACE(20, ("GOMP_critical_end: T#%d\n", gtid));
  74 #if OMPT_SUPPORT && OMPT_OPTIONAL
  75   OMPT_STORE_RETURN_ADDRESS(gtid);
  76 #endif
  77   __kmpc_end_critical(&loc, gtid, __kmp_unnamed_critical_addr);
  78 }
  79
  80 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_START)(void **pptr) {
  81   int gtid = __kmp_entry_gtid();
  82   MKLOC(loc, "GOMP_critical_name_start");
  83   KA_TRACE(20, ("GOMP_critical_name_start: T#%d\n", gtid));
  84   __kmpc_critical(&loc, gtid, (kmp_critical_name *)pptr);
  85 }
  86
  87 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_END)(void **pptr) {
  88   int gtid = __kmp_get_gtid();
  89   MKLOC(loc, "GOMP_critical_name_end");
  90   KA_TRACE(20, ("GOMP_critical_name_end: T#%d\n", gtid));
  91   __kmpc_end_critical(&loc, gtid, (kmp_critical_name *)pptr);
  92 }
  93
  94 // The Gnu codegen tries to use locked operations to perform atomic updates
  95 // inline.  If it can't, then it calls GOMP_atomic_start() before performing
  96 // the update and GOMP_atomic_end() afterward, regardless of the data type.
  97 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_START)(void) {
  98   int gtid = __kmp_entry_gtid();
  99   KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid));
 100
 101 #if OMPT_SUPPORT
 102   __ompt_thread_assign_wait_id(0);
 103 #endif
 104
 105   __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
 106 }
 107
 108 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_END)(void) {
 109   int gtid = __kmp_get_gtid();
 110   KA_TRACE(20, ("GOMP_atomic_end: T#%d\n", gtid));
 111   __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
 112 }
 113
 114 int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) {
 115   int gtid = __kmp_entry_gtid();
 116   MKLOC(loc, "GOMP_single_start");
 117   KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid));
 118
 119   if (!TCR_4(__kmp_init_parallel))
 120     __kmp_parallel_initialize();
 121   __kmp_resume_if_soft_paused();
 122
 123   // 3rd parameter == FALSE prevents kmp_enter_single from pushing a
 124   // workshare when USE_CHECKS is defined.  We need to avoid the push,
 125   // as there is no corresponding GOMP_single_end() call.
 126   kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE);
 127
 128 #if OMPT_SUPPORT && OMPT_OPTIONAL
 129   kmp_info_t *this_thr = __kmp_threads[gtid];
 130   kmp_team_t *team = this_thr->th.th_team;
 131   int tid = __kmp_tid_from_gtid(gtid);
 132
 133   if (ompt_enabled.enabled) {
 134     if (rc) {
 135       if (ompt_enabled.ompt_callback_work) {
 136         ompt_callbacks.ompt_callback(ompt_callback_work)(
 137             ompt_work_single_executor, ompt_scope_begin,
 138             &(team->t.ompt_team_info.parallel_data),
 139             &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
 140             1, OMPT_GET_RETURN_ADDRESS(0));
 141       }
 142     } else {
 143       if (ompt_enabled.ompt_callback_work) {
 144         ompt_callbacks.ompt_callback(ompt_callback_work)(
 145             ompt_work_single_other, ompt_scope_begin,
 146             &(team->t.ompt_team_info.parallel_data),
 147             &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
 148             1, OMPT_GET_RETURN_ADDRESS(0));
 149         ompt_callbacks.ompt_callback(ompt_callback_work)(
 150             ompt_work_single_other, ompt_scope_end,
 151             &(team->t.ompt_team_info.parallel_data),
 152             &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
 153             1, OMPT_GET_RETURN_ADDRESS(0));
 154       }
 155     }
 156   }
 157 #endif
 158
 159   return rc;
 160 }
 161
 162 void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) {
 163   void *retval;
 164   int gtid = __kmp_entry_gtid();
 165   MKLOC(loc, "GOMP_single_copy_start");
 166   KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid));
 167
 168   if (!TCR_4(__kmp_init_parallel))
 169     __kmp_parallel_initialize();
 170   __kmp_resume_if_soft_paused();
 171
 172   // If this is the first thread to enter, return NULL.  The generated code will
 173   // then call GOMP_single_copy_end() for this thread only, with the
 174   // copyprivate data pointer as an argument.
 175   if (__kmp_enter_single(gtid, &loc, FALSE))
 176     return NULL;
 177
 178 // Wait for the first thread to set the copyprivate data pointer,
 179 // and for all other threads to reach this point.
 180
 181 #if OMPT_SUPPORT && OMPT_OPTIONAL
 182   ompt_frame_t *ompt_frame;
 183   if (ompt_enabled.enabled) {
 184     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
 185     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 186     OMPT_STORE_RETURN_ADDRESS(gtid);
 187   }
 188 #endif
 189   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 190
 191   // Retrieve the value of the copyprivate data point, and wait for all
 192   // threads to do likewise, then return.
 193   retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data;
 194 #if OMPT_SUPPORT && OMPT_OPTIONAL
 195   if (ompt_enabled.enabled) {
 196     OMPT_STORE_RETURN_ADDRESS(gtid);
 197   }
 198 #endif
 199   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 200 #if OMPT_SUPPORT && OMPT_OPTIONAL
 201   if (ompt_enabled.enabled) {
 202     ompt_frame->enter_frame = ompt_data_none;
 203   }
 204 #endif
 205   return retval;
 206 }
 207
 208 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_END)(void *data) {
 209   int gtid = __kmp_get_gtid();
 210   KA_TRACE(20, ("GOMP_single_copy_end: T#%d\n", gtid));
 211
 212   // Set the copyprivate data pointer fo the team, then hit the barrier so that
 213   // the other threads will continue on and read it.  Hit another barrier before
 214   // continuing, so that the know that the copyprivate data pointer has been
 215   // propagated to all threads before trying to reuse the t_copypriv_data field.
 216   __kmp_team_from_gtid(gtid)->t.t_copypriv_data = data;
 217 #if OMPT_SUPPORT && OMPT_OPTIONAL
 218   ompt_frame_t *ompt_frame;
 219   if (ompt_enabled.enabled) {
 220     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
 221     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 222     OMPT_STORE_RETURN_ADDRESS(gtid);
 223   }
 224 #endif
 225   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 226 #if OMPT_SUPPORT && OMPT_OPTIONAL
 227   if (ompt_enabled.enabled) {
 228     OMPT_STORE_RETURN_ADDRESS(gtid);
 229   }
 230 #endif
 231   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 232 #if OMPT_SUPPORT && OMPT_OPTIONAL
 233   if (ompt_enabled.enabled) {
 234     ompt_frame->enter_frame = ompt_data_none;
 235   }
 236 #endif
 237 }
 238
 239 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_START)(void) {
 240   int gtid = __kmp_entry_gtid();
 241   MKLOC(loc, "GOMP_ordered_start");
 242   KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid));
 243 #if OMPT_SUPPORT && OMPT_OPTIONAL
 244   OMPT_STORE_RETURN_ADDRESS(gtid);
 245 #endif
 246   __kmpc_ordered(&loc, gtid);
 247 }
 248
 249 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) {
 250   int gtid = __kmp_get_gtid();
 251   MKLOC(loc, "GOMP_ordered_end");
 252   KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid));
 253 #if OMPT_SUPPORT && OMPT_OPTIONAL
 254   OMPT_STORE_RETURN_ADDRESS(gtid);
 255 #endif
 256   __kmpc_end_ordered(&loc, gtid);
 257 }
 258
 259 // Dispatch macro defs
 260 //
 261 // They come in two flavors: 64-bit unsigned, and either 32-bit signed
 262 // (IA-32 architecture) or 64-bit signed (Intel(R) 64).
 263
 264 #if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS
 265 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4
 266 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4
 267 #define KMP_DISPATCH_NEXT __kmpc_dispatch_next_4
 268 #else
 269 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_8
 270 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_8
 271 #define KMP_DISPATCH_NEXT __kmpc_dispatch_next_8
 272 #endif /* KMP_ARCH_X86 */
 273
 274 #define KMP_DISPATCH_INIT_ULL __kmp_aux_dispatch_init_8u
 275 #define KMP_DISPATCH_FINI_CHUNK_ULL __kmp_aux_dispatch_fini_chunk_8u
 276 #define KMP_DISPATCH_NEXT_ULL __kmpc_dispatch_next_8u
 277
 278 // The parallel contruct
 279
 280 #ifndef KMP_DEBUG
 281 static
 282 #endif /* KMP_DEBUG */
 283     void
 284     __kmp_GOMP_microtask_wrapper(int *gtid, int *npr, void (*task)(void *),
 285                                  void *data) {
 286 #if OMPT_SUPPORT
 287   kmp_info_t *thr;
 288   ompt_frame_t *ompt_frame;
 289   ompt_state_t enclosing_state;
 290
 291   if (ompt_enabled.enabled) {
 292     // get pointer to thread data structure
 293     thr = __kmp_threads[*gtid];
 294
 295     // save enclosing task state; set current state for task
 296     enclosing_state = thr->th.ompt_thread_info.state;
 297     thr->th.ompt_thread_info.state = ompt_state_work_parallel;
 298
 299     // set task frame
 300     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
 301     ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 302   }
 303 #endif
 304
 305   task(data);
 306
 307 #if OMPT_SUPPORT
 308   if (ompt_enabled.enabled) {
 309     // clear task frame
 310     ompt_frame->exit_frame = ompt_data_none;
 311
 312     // restore enclosing state
 313     thr->th.ompt_thread_info.state = enclosing_state;
 314   }
 315 #endif
 316 }
 317
 318 #ifndef KMP_DEBUG
 319 static
 320 #endif /* KMP_DEBUG */
 321     void
 322     __kmp_GOMP_parallel_microtask_wrapper(int *gtid, int *npr,
 323                                           void (*task)(void *), void *data,
 324                                           unsigned num_threads, ident_t *loc,
 325                                           enum sched_type schedule, long start,
 326                                           long end, long incr,
 327                                           long chunk_size) {
 328   // Intialize the loop worksharing construct.
 329
 330   KMP_DISPATCH_INIT(loc, *gtid, schedule, start, end, incr, chunk_size,
 331                     schedule != kmp_sch_static);
 332
 333 #if OMPT_SUPPORT
 334   kmp_info_t *thr;
 335   ompt_frame_t *ompt_frame;
 336   ompt_state_t enclosing_state;
 337
 338   if (ompt_enabled.enabled) {
 339     thr = __kmp_threads[*gtid];
 340     // save enclosing task state; set current state for task
 341     enclosing_state = thr->th.ompt_thread_info.state;
 342     thr->th.ompt_thread_info.state = ompt_state_work_parallel;
 343
 344     // set task frame
 345     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
 346     ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 347   }
 348 #endif
 349
 350   // Now invoke the microtask.
 351   task(data);
 352
 353 #if OMPT_SUPPORT
 354   if (ompt_enabled.enabled) {
 355     // clear task frame
 356     ompt_frame->exit_frame = ompt_data_none;
 357
 358     // reset enclosing state
 359     thr->th.ompt_thread_info.state = enclosing_state;
 360   }
 361 #endif
 362 }
 363
 364 #ifndef KMP_DEBUG
 365 static
 366 #endif /* KMP_DEBUG */
 367     void
 368     __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *),
 369                          microtask_t wrapper, int argc, ...) {
 370   int rc;
 371   kmp_info_t *thr = __kmp_threads[gtid];
 372   kmp_team_t *team = thr->th.th_team;
 373   int tid = __kmp_tid_from_gtid(gtid);
 374
 375   va_list ap;
 376   va_start(ap, argc);
 377
 378   rc = __kmp_fork_call(loc, gtid, fork_context_gnu, argc, wrapper,
 379                        __kmp_invoke_task_func,
 380 #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
 381                        &ap
 382 #else
 383                        ap
 384 #endif
 385                        );
 386
 387   va_end(ap);
 388
 389   if (rc) {
 390     __kmp_run_before_invoked_task(gtid, tid, thr, team);
 391   }
 392
 393 #if OMPT_SUPPORT
 394   int ompt_team_size;
 395   if (ompt_enabled.enabled) {
 396     ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
 397     ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
 398
 399     // implicit task callback
 400     if (ompt_enabled.ompt_callback_implicit_task) {
 401       ompt_team_size = __kmp_team_from_gtid(gtid)->t.t_nproc;
 402       ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
 403           ompt_scope_begin, &(team_info->parallel_data),
 404           &(task_info->task_data), ompt_team_size, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
 405       task_info->thread_num = __kmp_tid_from_gtid(gtid);
 406     }
 407     thr->th.ompt_thread_info.state = ompt_state_work_parallel;
 408   }
 409 #endif
 410 }
 411
 412 static void __kmp_GOMP_serialized_parallel(ident_t *loc, kmp_int32 gtid,
 413                                            void (*task)(void *)) {
 414 #if OMPT_SUPPORT
 415   OMPT_STORE_RETURN_ADDRESS(gtid);
 416 #endif
 417   __kmp_serialized_parallel(loc, gtid);
 418 }
 419
 420 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *),
 421                                                        void *data,
 422                                                        unsigned num_threads) {
 423   int gtid = __kmp_entry_gtid();
 424
 425 #if OMPT_SUPPORT
 426   ompt_frame_t *parent_frame, *frame;
 427
 428   if (ompt_enabled.enabled) {
 429     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);
 430     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 431     OMPT_STORE_RETURN_ADDRESS(gtid);
 432   }
 433 #endif
 434
 435   MKLOC(loc, "GOMP_parallel_start");
 436   KA_TRACE(20, ("GOMP_parallel_start: T#%d\n", gtid));
 437
 438   if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
 439     if (num_threads != 0) {
 440       __kmp_push_num_threads(&loc, gtid, num_threads);
 441     }
 442     __kmp_GOMP_fork_call(&loc, gtid, task,
 443                          (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task,
 444                          data);
 445   } else {
 446     __kmp_GOMP_serialized_parallel(&loc, gtid, task);
 447   }
 448
 449 #if OMPT_SUPPORT
 450   if (ompt_enabled.enabled) {
 451     __ompt_get_task_info_internal(0, NULL, NULL, &frame, NULL, NULL);
 452     frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 453   }
 454 #endif
 455 }
 456
 457 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(void) {
 458   int gtid = __kmp_get_gtid();
 459   kmp_info_t *thr;
 460
 461   thr = __kmp_threads[gtid];
 462
 463   MKLOC(loc, "GOMP_parallel_end");
 464   KA_TRACE(20, ("GOMP_parallel_end: T#%d\n", gtid));
 465
 466   if (!thr->th.th_team->t.t_serialized) {
 467     __kmp_run_after_invoked_task(gtid, __kmp_tid_from_gtid(gtid), thr,
 468                                  thr->th.th_team);
 469
 470 #if OMPT_SUPPORT
 471     if (ompt_enabled.enabled) {
 472       // Implicit task is finished here, in the barrier we might schedule
 473       // deferred tasks,
 474       // these don't see the implicit task on the stack
 475       OMPT_CUR_TASK_INFO(thr)->frame.exit_frame = ompt_data_none;
 476     }
 477 #endif
 478
 479     __kmp_join_call(&loc, gtid
 480 #if OMPT_SUPPORT
 481                     ,
 482                     fork_context_gnu
 483 #endif
 484                     );
 485   } else {
 486     __kmpc_end_serialized_parallel(&loc, gtid);
 487   }
 488 }
 489
 490 // Loop worksharing constructs
 491
 492 // The Gnu codegen passes in an exclusive upper bound for the overall range,
 493 // but the libguide dispatch code expects an inclusive upper bound, hence the
 494 // "end - incr" 5th argument to KMP_DISPATCH_INIT (and the " ub - str" 11th
 495 // argument to __kmp_GOMP_fork_call).
 496 //
 497 // Conversely, KMP_DISPATCH_NEXT returns and inclusive upper bound in *p_ub,
 498 // but the Gnu codegen expects an excluside upper bound, so the adjustment
 499 // "*p_ub += stride" compenstates for the discrepancy.
 500 //
 501 // Correction: the gnu codegen always adjusts the upper bound by +-1, not the
 502 // stride value.  We adjust the dispatch parameters accordingly (by +-1), but
 503 // we still adjust p_ub by the actual stride value.
 504 //
 505 // The "runtime" versions do not take a chunk_sz parameter.
 506 //
 507 // The profile lib cannot support construct checking of unordered loops that
 508 // are predetermined by the compiler to be statically scheduled, as the gcc
 509 // codegen will not always emit calls to GOMP_loop_static_next() to get the
 510 // next iteration.  Instead, it emits inline code to call omp_get_thread_num()
 511 // num and calculate the iteration space using the result.  It doesn't do this
 512 // with ordered static loop, so they can be checked.
 513
 514 #if OMPT_SUPPORT
 515 #define IF_OMPT_SUPPORT(code) code
 516 #else
 517 #define IF_OMPT_SUPPORT(code)
 518 #endif
 519
 520 #define LOOP_START(func, schedule)                                             \
 521   int func(long lb, long ub, long str, long chunk_sz, long *p_lb,              \
 522            long *p_ub) {                                                       \
 523     int status;                                                                \
 524     long stride;                                                               \
 525     int gtid = __kmp_entry_gtid();                                             \
 526     MKLOC(loc, KMP_STR(func));                                                 \
 527     KA_TRACE(                                                                  \
 528         20,                                                                    \
 529         (KMP_STR(                                                              \
 530              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
 531          gtid, lb, ub, str, chunk_sz));                                        \
 532                                                                                \
 533     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 534       IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
 535       KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
 536                         (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,        \
 537                         (schedule) != kmp_sch_static);                         \
 538       IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
 539       status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,            \
 540                                  (kmp_int *)p_ub, (kmp_int *)&stride);         \
 541       if (status) {                                                            \
 542         KMP_DEBUG_ASSERT(stride == str);                                       \
 543         *p_ub += (str > 0) ? 1 : -1;                                           \
 544       }                                                                        \
 545     } else {                                                                   \
 546       status = 0;                                                              \
 547     }                                                                          \
 548                                                                                \
 549     KA_TRACE(                                                                  \
 550         20,                                                                    \
 551         (KMP_STR(                                                              \
 552              func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n",    \
 553          gtid, *p_lb, *p_ub, status));                                         \
 554     return status;                                                             \
 555   }
 556
 557 #define LOOP_RUNTIME_START(func, schedule)                                     \
 558   int func(long lb, long ub, long str, long *p_lb, long *p_ub) {               \
 559     int status;                                                                \
 560     long stride;                                                               \
 561     long chunk_sz = 0;                                                         \
 562     int gtid = __kmp_entry_gtid();                                             \
 563     MKLOC(loc, KMP_STR(func));                                                 \
 564     KA_TRACE(                                                                  \
 565         20,                                                                    \
 566         (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \
 567          gtid, lb, ub, str, chunk_sz));                                        \
 568                                                                                \
 569     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 570       IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
 571       KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
 572                         (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \
 573       IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
 574       status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,            \
 575                                  (kmp_int *)p_ub, (kmp_int *)&stride);         \
 576       if (status) {                                                            \
 577         KMP_DEBUG_ASSERT(stride == str);                                       \
 578         *p_ub += (str > 0) ? 1 : -1;                                           \
 579       }                                                                        \
 580     } else {                                                                   \
 581       status = 0;                                                              \
 582     }                                                                          \
 583                                                                                \
 584     KA_TRACE(                                                                  \
 585         20,                                                                    \
 586         (KMP_STR(                                                              \
 587              func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n",    \
 588          gtid, *p_lb, *p_ub, status));                                         \
 589     return status;                                                             \
 590   }
 591
 592 #define KMP_DOACROSS_FINI(status, gtid)                                        \
 593   if (!status && __kmp_threads[gtid]->th.th_dispatch->th_doacross_flags) {     \
 594     __kmpc_doacross_fini(NULL, gtid);                                          \
 595   }
 596
 597 #define LOOP_NEXT(func, fini_code)                                             \
 598   int func(long *p_lb, long *p_ub) {                                           \
 599     int status;                                                                \
 600     long stride;                                                               \
 601     int gtid = __kmp_get_gtid();                                               \
 602     MKLOC(loc, KMP_STR(func));                                                 \
 603     KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid));                            \
 604                                                                                \
 605     IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
 606     fini_code status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,    \
 607                                          (kmp_int *)p_ub, (kmp_int *)&stride); \
 608     if (status) {                                                              \
 609       *p_ub += (stride > 0) ? 1 : -1;                                          \
 610     }                                                                          \
 611     KMP_DOACROSS_FINI(status, gtid)                                            \
 612                                                                                \
 613     KA_TRACE(                                                                  \
 614         20,                                                                    \
 615         (KMP_STR(func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, stride 0x%lx, " \
 616                        "returning %d\n",                                       \
 617          gtid, *p_lb, *p_ub, stride, status));                                 \
 618     return status;                                                             \
 619   }
 620
 621 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_START), kmp_sch_static)
 622 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT), {})
 623 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START),
 624            kmp_sch_dynamic_chunked)
 625 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT), {})
 626 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_START),
 627            kmp_sch_guided_chunked)
 628 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT), {})
 629 LOOP_RUNTIME_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_START),
 630                    kmp_sch_runtime)
 631 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT), {})
 632
 633 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START),
 634            kmp_ord_static)
 635 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT),
 636           { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
 637 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START),
 638            kmp_ord_dynamic_chunked)
 639 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT),
 640           { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
 641 LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START),
 642            kmp_ord_guided_chunked)
 643 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT),
 644           { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
 645 LOOP_RUNTIME_START(
 646     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START),
 647     kmp_ord_runtime)
 648 LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT),
 649           { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
 650
 651 #define LOOP_DOACROSS_START(func, schedule)                                    \
 652   bool func(unsigned ncounts, long *counts, long chunk_sz, long *p_lb,         \
 653             long *p_ub) {                                                      \
 654     int status;                                                                \
 655     long stride, lb, ub, str;                                                  \
 656     int gtid = __kmp_entry_gtid();                                             \
 657     struct kmp_dim *dims =                                                     \
 658         (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts);    \
 659     MKLOC(loc, KMP_STR(func));                                                 \
 660     for (unsigned i = 0; i < ncounts; ++i) {                                   \
 661       dims[i].lo = 0;                                                          \
 662       dims[i].up = counts[i] - 1;                                              \
 663       dims[i].st = 1;                                                          \
 664     }                                                                          \
 665     __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims);                      \
 666     lb = 0;                                                                    \
 667     ub = counts[0];                                                            \
 668     str = 1;                                                                   \
 669     KA_TRACE(20, (KMP_STR(func) ": T#%d, ncounts %u, lb 0x%lx, ub 0x%lx, str " \
 670                                 "0x%lx, chunk_sz "                             \
 671                                 "0x%lx\n",                                     \
 672                   gtid, ncounts, lb, ub, str, chunk_sz));                      \
 673                                                                                \
 674     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 675       KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
 676                         (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,        \
 677                         (schedule) != kmp_sch_static);                         \
 678       status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,            \
 679                                  (kmp_int *)p_ub, (kmp_int *)&stride);         \
 680       if (status) {                                                            \
 681         KMP_DEBUG_ASSERT(stride == str);                                       \
 682         *p_ub += (str > 0) ? 1 : -1;                                           \
 683       }                                                                        \
 684     } else {                                                                   \
 685       status = 0;                                                              \
 686     }                                                                          \
 687     KMP_DOACROSS_FINI(status, gtid);                                           \
 688                                                                                \
 689     KA_TRACE(                                                                  \
 690         20,                                                                    \
 691         (KMP_STR(                                                              \
 692              func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n",    \
 693          gtid, *p_lb, *p_ub, status));                                         \
 694     __kmp_free(dims);                                                          \
 695     return status;                                                             \
 696   }
 697
 698 #define LOOP_DOACROSS_RUNTIME_START(func, schedule)                            \
 699   int func(unsigned ncounts, long *counts, long *p_lb, long *p_ub) {           \
 700     int status;                                                                \
 701     long stride, lb, ub, str;                                                  \
 702     long chunk_sz = 0;                                                         \
 703     int gtid = __kmp_entry_gtid();                                             \
 704     struct kmp_dim *dims =                                                     \
 705         (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts);    \
 706     MKLOC(loc, KMP_STR(func));                                                 \
 707     for (unsigned i = 0; i < ncounts; ++i) {                                   \
 708       dims[i].lo = 0;                                                          \
 709       dims[i].up = counts[i] - 1;                                              \
 710       dims[i].st = 1;                                                          \
 711     }                                                                          \
 712     __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims);                      \
 713     lb = 0;                                                                    \
 714     ub = counts[0];                                                            \
 715     str = 1;                                                                   \
 716     KA_TRACE(                                                                  \
 717         20,                                                                    \
 718         (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \
 719          gtid, lb, ub, str, chunk_sz));                                        \
 720                                                                                \
 721     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 722       KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
 723                         (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \
 724       status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb,            \
 725                                  (kmp_int *)p_ub, (kmp_int *)&stride);         \
 726       if (status) {                                                            \
 727         KMP_DEBUG_ASSERT(stride == str);                                       \
 728         *p_ub += (str > 0) ? 1 : -1;                                           \
 729       }                                                                        \
 730     } else {                                                                   \
 731       status = 0;                                                              \
 732     }                                                                          \
 733     KMP_DOACROSS_FINI(status, gtid);                                           \
 734                                                                                \
 735     KA_TRACE(                                                                  \
 736         20,                                                                    \
 737         (KMP_STR(                                                              \
 738              func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n",    \
 739          gtid, *p_lb, *p_ub, status));                                         \
 740     __kmp_free(dims);                                                          \
 741     return status;                                                             \
 742   }
 743
 744 LOOP_DOACROSS_START(
 745     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START),
 746     kmp_sch_static)
 747 LOOP_DOACROSS_START(
 748     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START),
 749     kmp_sch_dynamic_chunked)
 750 LOOP_DOACROSS_START(
 751     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START),
 752     kmp_sch_guided_chunked)
 753 LOOP_DOACROSS_RUNTIME_START(
 754     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START),
 755     kmp_sch_runtime)
 756
 757 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END)(void) {
 758   int gtid = __kmp_get_gtid();
 759   KA_TRACE(20, ("GOMP_loop_end: T#%d\n", gtid))
 760
 761 #if OMPT_SUPPORT && OMPT_OPTIONAL
 762   ompt_frame_t *ompt_frame;
 763   if (ompt_enabled.enabled) {
 764     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
 765     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
 766     OMPT_STORE_RETURN_ADDRESS(gtid);
 767   }
 768 #endif
 769   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 770 #if OMPT_SUPPORT && OMPT_OPTIONAL
 771   if (ompt_enabled.enabled) {
 772     ompt_frame->enter_frame = ompt_data_none;
 773   }
 774 #endif
 775
 776   KA_TRACE(20, ("GOMP_loop_end exit: T#%d\n", gtid))
 777 }
 778
 779 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_NOWAIT)(void) {
 780   KA_TRACE(20, ("GOMP_loop_end_nowait: T#%d\n", __kmp_get_gtid()))
 781 }
 782
 783 // Unsigned long long loop worksharing constructs
 784 //
 785 // These are new with gcc 4.4
 786
 787 #define LOOP_START_ULL(func, schedule)                                         \
 788   int func(int up, unsigned long long lb, unsigned long long ub,               \
 789            unsigned long long str, unsigned long long chunk_sz,                \
 790            unsigned long long *p_lb, unsigned long long *p_ub) {               \
 791     int status;                                                                \
 792     long long str2 = up ? ((long long)str) : -((long long)str);                \
 793     long long stride;                                                          \
 794     int gtid = __kmp_entry_gtid();                                             \
 795     MKLOC(loc, KMP_STR(func));                                                 \
 796                                                                                \
 797     KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str "    \
 798                                 "0x%llx, chunk_sz 0x%llx\n",                   \
 799                   gtid, up, lb, ub, str, chunk_sz));                           \
 800                                                                                \
 801     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 802       KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb,                        \
 803                             (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz,  \
 804                             (schedule) != kmp_sch_static);                     \
 805       status =                                                                 \
 806           KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb,          \
 807                                 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride);     \
 808       if (status) {                                                            \
 809         KMP_DEBUG_ASSERT(stride == str2);                                      \
 810         *p_ub += (str > 0) ? 1 : -1;                                           \
 811       }                                                                        \
 812     } else {                                                                   \
 813       status = 0;                                                              \
 814     }                                                                          \
 815                                                                                \
 816     KA_TRACE(                                                                  \
 817         20,                                                                    \
 818         (KMP_STR(                                                              \
 819              func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n",  \
 820          gtid, *p_lb, *p_ub, status));                                         \
 821     return status;                                                             \
 822   }
 823
 824 #define LOOP_RUNTIME_START_ULL(func, schedule)                                 \
 825   int func(int up, unsigned long long lb, unsigned long long ub,               \
 826            unsigned long long str, unsigned long long *p_lb,                   \
 827            unsigned long long *p_ub) {                                         \
 828     int status;                                                                \
 829     long long str2 = up ? ((long long)str) : -((long long)str);                \
 830     unsigned long long stride;                                                 \
 831     unsigned long long chunk_sz = 0;                                           \
 832     int gtid = __kmp_entry_gtid();                                             \
 833     MKLOC(loc, KMP_STR(func));                                                 \
 834                                                                                \
 835     KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str "    \
 836                                 "0x%llx, chunk_sz 0x%llx\n",                   \
 837                   gtid, up, lb, ub, str, chunk_sz));                           \
 838                                                                                \
 839     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 840       KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb,                        \
 841                             (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz,  \
 842                             TRUE);                                             \
 843       status =                                                                 \
 844           KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb,          \
 845                                 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride);     \
 846       if (status) {                                                            \
 847         KMP_DEBUG_ASSERT((long long)stride == str2);                           \
 848         *p_ub += (str > 0) ? 1 : -1;                                           \
 849       }                                                                        \
 850     } else {                                                                   \
 851       status = 0;                                                              \
 852     }                                                                          \
 853                                                                                \
 854     KA_TRACE(                                                                  \
 855         20,                                                                    \
 856         (KMP_STR(                                                              \
 857              func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n",  \
 858          gtid, *p_lb, *p_ub, status));                                         \
 859     return status;                                                             \
 860   }
 861
 862 #define LOOP_NEXT_ULL(func, fini_code)                                         \
 863   int func(unsigned long long *p_lb, unsigned long long *p_ub) {               \
 864     int status;                                                                \
 865     long long stride;                                                          \
 866     int gtid = __kmp_get_gtid();                                               \
 867     MKLOC(loc, KMP_STR(func));                                                 \
 868     KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid));                            \
 869                                                                                \
 870     fini_code status =                                                         \
 871         KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb,            \
 872                               (kmp_uint64 *)p_ub, (kmp_int64 *)&stride);       \
 873     if (status) {                                                              \
 874       *p_ub += (stride > 0) ? 1 : -1;                                          \
 875     }                                                                          \
 876                                                                                \
 877     KA_TRACE(                                                                  \
 878         20,                                                                    \
 879         (KMP_STR(                                                              \
 880              func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, stride 0x%llx, "  \
 881                    "returning %d\n",                                           \
 882          gtid, *p_lb, *p_ub, stride, status));                                 \
 883     return status;                                                             \
 884   }
 885
 886 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START),
 887                kmp_sch_static)
 888 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT), {})
 889 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START),
 890                kmp_sch_dynamic_chunked)
 891 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT), {})
 892 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START),
 893                kmp_sch_guided_chunked)
 894 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT), {})
 895 LOOP_RUNTIME_START_ULL(
 896     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START), kmp_sch_runtime)
 897 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT), {})
 898
 899 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START),
 900                kmp_ord_static)
 901 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT),
 902               { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
 903 LOOP_START_ULL(
 904     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START),
 905     kmp_ord_dynamic_chunked)
 906 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT),
 907               { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
 908 LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START),
 909                kmp_ord_guided_chunked)
 910 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT),
 911               { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
 912 LOOP_RUNTIME_START_ULL(
 913     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START),
 914     kmp_ord_runtime)
 915 LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT),
 916               { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
 917
 918 #define LOOP_DOACROSS_START_ULL(func, schedule)                                \
 919   int func(unsigned ncounts, unsigned long long *counts,                       \
 920            unsigned long long chunk_sz, unsigned long long *p_lb,              \
 921            unsigned long long *p_ub) {                                         \
 922     int status;                                                                \
 923     long long stride, str, lb, ub;                                             \
 924     int gtid = __kmp_entry_gtid();                                             \
 925     struct kmp_dim *dims =                                                     \
 926         (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts);    \
 927     MKLOC(loc, KMP_STR(func));                                                 \
 928     for (unsigned i = 0; i < ncounts; ++i) {                                   \
 929       dims[i].lo = 0;                                                          \
 930       dims[i].up = counts[i] - 1;                                              \
 931       dims[i].st = 1;                                                          \
 932     }                                                                          \
 933     __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims);                      \
 934     lb = 0;                                                                    \
 935     ub = counts[0];                                                            \
 936     str = 1;                                                                   \
 937                                                                                \
 938     KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str "           \
 939                                 "0x%llx, chunk_sz 0x%llx\n",                   \
 940                   gtid, lb, ub, str, chunk_sz));                               \
 941                                                                                \
 942     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 943       KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb,                        \
 944                             (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,    \
 945                             (schedule) != kmp_sch_static);                     \
 946       status =                                                                 \
 947           KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb,          \
 948                                 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride);     \
 949       if (status) {                                                            \
 950         KMP_DEBUG_ASSERT(stride == str);                                       \
 951         *p_ub += (str > 0) ? 1 : -1;                                           \
 952       }                                                                        \
 953     } else {                                                                   \
 954       status = 0;                                                              \
 955     }                                                                          \
 956     KMP_DOACROSS_FINI(status, gtid);                                           \
 957                                                                                \
 958     KA_TRACE(                                                                  \
 959         20,                                                                    \
 960         (KMP_STR(                                                              \
 961              func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n",  \
 962          gtid, *p_lb, *p_ub, status));                                         \
 963     __kmp_free(dims);                                                          \
 964     return status;                                                             \
 965   }
 966
 967 #define LOOP_DOACROSS_RUNTIME_START_ULL(func, schedule)                        \
 968   int func(unsigned ncounts, unsigned long long *counts,                       \
 969            unsigned long long *p_lb, unsigned long long *p_ub) {               \
 970     int status;                                                                \
 971     unsigned long long stride, str, lb, ub;                                    \
 972     unsigned long long chunk_sz = 0;                                           \
 973     int gtid = __kmp_entry_gtid();                                             \
 974     struct kmp_dim *dims =                                                     \
 975         (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts);    \
 976     MKLOC(loc, KMP_STR(func));                                                 \
 977     for (unsigned i = 0; i < ncounts; ++i) {                                   \
 978       dims[i].lo = 0;                                                          \
 979       dims[i].up = counts[i] - 1;                                              \
 980       dims[i].st = 1;                                                          \
 981     }                                                                          \
 982     __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims);                      \
 983     lb = 0;                                                                    \
 984     ub = counts[0];                                                            \
 985     str = 1;                                                                   \
 986     KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str "           \
 987                                 "0x%llx, chunk_sz 0x%llx\n",                   \
 988                   gtid, lb, ub, str, chunk_sz));                               \
 989                                                                                \
 990     if ((str > 0) ? (lb < ub) : (lb > ub)) {                                   \
 991       KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb,                        \
 992                             (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,    \
 993                             TRUE);                                             \
 994       status =                                                                 \
 995           KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb,          \
 996                                 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride);     \
 997       if (status) {                                                            \
 998         KMP_DEBUG_ASSERT(stride == str);                                       \
 999         *p_ub += (str > 0) ? 1 : -1;                                           \
1000       }                                                                        \
1001     } else {                                                                   \
1002       status = 0;                                                              \
1003     }                                                                          \
1004     KMP_DOACROSS_FINI(status, gtid);                                           \
1005                                                                                \
1006     KA_TRACE(                                                                  \
1007         20,                                                                    \
1008         (KMP_STR(                                                              \
1009              func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n",  \
1010          gtid, *p_lb, *p_ub, status));                                         \
1011     __kmp_free(dims);                                                          \
1012     return status;                                                             \
1013   }
1014
1015 LOOP_DOACROSS_START_ULL(
1016     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START),
1017     kmp_sch_static)
1018 LOOP_DOACROSS_START_ULL(
1019     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START),
1020     kmp_sch_dynamic_chunked)
1021 LOOP_DOACROSS_START_ULL(
1022     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START),
1023     kmp_sch_guided_chunked)
1024 LOOP_DOACROSS_RUNTIME_START_ULL(
1025     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START),
1026     kmp_sch_runtime)
1027
1028 // Combined parallel / loop worksharing constructs
1029 //
1030 // There are no ull versions (yet).
1031
1032 #define PARALLEL_LOOP_START(func, schedule, ompt_pre, ompt_post)               \
1033   void func(void (*task)(void *), void *data, unsigned num_threads, long lb,   \
1034             long ub, long str, long chunk_sz) {                                \
1035     int gtid = __kmp_entry_gtid();                                             \
1036     MKLOC(loc, KMP_STR(func));                                                 \
1037     KA_TRACE(                                                                  \
1038         20,                                                                    \
1039         (KMP_STR(                                                              \
1040              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
1041          gtid, lb, ub, str, chunk_sz));                                        \
1042                                                                                \
1043     ompt_pre();                                                                \
1044                                                                                \
1045     if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {                       \
1046       if (num_threads != 0) {                                                  \
1047         __kmp_push_num_threads(&loc, gtid, num_threads);                       \
1048       }                                                                        \
1049       __kmp_GOMP_fork_call(&loc, gtid, task,                                   \
1050                            (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \
1051                            9, task, data, num_threads, &loc, (schedule), lb,   \
1052                            (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz);    \
1053       IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid));                        \
1054     } else {                                                                   \
1055       __kmp_GOMP_serialized_parallel(&loc, gtid, task);                        \
1056       IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid));                        \
1057     }                                                                          \
1058                                                                                \
1059     KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                              \
1060                       (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,          \
1061                       (schedule) != kmp_sch_static);                           \
1062                                                                                \
1063     ompt_post();                                                               \
1064                                                                                \
1065     KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid));                       \
1066   }
1067
1068 #if OMPT_SUPPORT && OMPT_OPTIONAL
1069
1070 #define OMPT_LOOP_PRE()                                                        \
1071   ompt_frame_t *parent_frame;                                                  \
1072   if (ompt_enabled.enabled) {                                                  \
1073     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);   \
1074     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);                 \
1075     OMPT_STORE_RETURN_ADDRESS(gtid);                                           \
1076   }
1077
1078 #define OMPT_LOOP_POST()                                                       \
1079   if (ompt_enabled.enabled) {                                                  \
1080     parent_frame->enter_frame = ompt_data_none;                                \
1081   }
1082
1083 #else
1084
1085 #define OMPT_LOOP_PRE()
1086
1087 #define OMPT_LOOP_POST()
1088
1089 #endif
1090
1091 PARALLEL_LOOP_START(
1092     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START),
1093     kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1094 PARALLEL_LOOP_START(
1095     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START),
1096     kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1097 PARALLEL_LOOP_START(
1098     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START),
1099     kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1100 PARALLEL_LOOP_START(
1101     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START),
1102     kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1103
1104 // Tasking constructs
1105
1106 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data,
1107                                              void (*copy_func)(void *, void *),
1108                                              long arg_size, long arg_align,
1109                                              bool if_cond, unsigned gomp_flags,
1110                                              void **depend) {
1111   MKLOC(loc, "GOMP_task");
1112   int gtid = __kmp_entry_gtid();
1113   kmp_int32 flags = 0;
1114   kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1115
1116   KA_TRACE(20, ("GOMP_task: T#%d\n", gtid));
1117
1118   // The low-order bit is the "untied" flag
1119   if (!(gomp_flags & 1)) {
1120     input_flags->tiedness = 1;
1121   }
1122   // The second low-order bit is the "final" flag
1123   if (gomp_flags & 2) {
1124     input_flags->final = 1;
1125   }
1126   input_flags->native = 1;
1127   // __kmp_task_alloc() sets up all other flags
1128
1129   if (!if_cond) {
1130     arg_size = 0;
1131   }
1132
1133   kmp_task_t *task = __kmp_task_alloc(
1134       &loc, gtid, input_flags, sizeof(kmp_task_t),
1135       arg_size ? arg_size + arg_align - 1 : 0, (kmp_routine_entry_t)func);
1136
1137   if (arg_size > 0) {
1138     if (arg_align > 0) {
1139       task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) /
1140                                arg_align * arg_align);
1141     }
1142     // else error??
1143
1144     if (copy_func) {
1145       (*copy_func)(task->shareds, data);
1146     } else {
1147       KMP_MEMCPY(task->shareds, data, arg_size);
1148     }
1149   }
1150
1151 #if OMPT_SUPPORT
1152   kmp_taskdata_t *current_task;
1153   if (ompt_enabled.enabled) {
1154     OMPT_STORE_RETURN_ADDRESS(gtid);
1155     current_task = __kmp_threads[gtid]->th.th_current_task;
1156     current_task->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1157   }
1158 #endif
1159
1160   if (if_cond) {
1161     if (gomp_flags & 8) {
1162       KMP_ASSERT(depend);
1163       const size_t ndeps = (kmp_intptr_t)depend[0];
1164       const size_t nout = (kmp_intptr_t)depend[1];
1165       kmp_depend_info_t dep_list[ndeps];
1166
1167       for (size_t i = 0U; i < ndeps; i++) {
1168         dep_list[i].base_addr = (kmp_intptr_t)depend[2U + i];
1169         dep_list[i].len = 0U;
1170         dep_list[i].flags.in = 1;
1171         dep_list[i].flags.out = (i < nout);
1172       }
1173       __kmpc_omp_task_with_deps(&loc, gtid, task, ndeps, dep_list, 0, NULL);
1174     } else {
1175       __kmpc_omp_task(&loc, gtid, task);
1176     }
1177   } else {
1178 #if OMPT_SUPPORT
1179     ompt_thread_info_t oldInfo;
1180     kmp_info_t *thread;
1181     kmp_taskdata_t *taskdata;
1182     if (ompt_enabled.enabled) {
1183       // Store the threads states and restore them after the task
1184       thread = __kmp_threads[gtid];
1185       taskdata = KMP_TASK_TO_TASKDATA(task);
1186       oldInfo = thread->th.ompt_thread_info;
1187       thread->th.ompt_thread_info.wait_id = 0;
1188       thread->th.ompt_thread_info.state = ompt_state_work_parallel;
1189       taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1190       OMPT_STORE_RETURN_ADDRESS(gtid);
1191     }
1192 #endif
1193
1194     __kmpc_omp_task_begin_if0(&loc, gtid, task);
1195     func(data);
1196     __kmpc_omp_task_complete_if0(&loc, gtid, task);
1197
1198 #if OMPT_SUPPORT
1199     if (ompt_enabled.enabled) {
1200       thread->th.ompt_thread_info = oldInfo;
1201       taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
1202     }
1203 #endif
1204   }
1205 #if OMPT_SUPPORT
1206   if (ompt_enabled.enabled) {
1207     current_task->ompt_task_info.frame.enter_frame = ompt_data_none;
1208   }
1209 #endif
1210
1211   KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid));
1212 }
1213
1214 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT)(void) {
1215   MKLOC(loc, "GOMP_taskwait");
1216   int gtid = __kmp_entry_gtid();
1217
1218 #if OMPT_SUPPORT
1219   if (ompt_enabled.enabled)
1220     OMPT_STORE_RETURN_ADDRESS(gtid);
1221 #endif
1222
1223   KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid));
1224
1225   __kmpc_omp_taskwait(&loc, gtid);
1226
1227   KA_TRACE(20, ("GOMP_taskwait exit: T#%d\n", gtid));
1228 }
1229
1230 // Sections worksharing constructs
1231 //
1232 // For the sections construct, we initialize a dynamically scheduled loop
1233 // worksharing construct with lb 1 and stride 1, and use the iteration #'s
1234 // that its returns as sections ids.
1235 //
1236 // There are no special entry points for ordered sections, so we always use
1237 // the dynamically scheduled workshare, even if the sections aren't ordered.
1238
1239 unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_START)(unsigned count) {
1240   int status;
1241   kmp_int lb, ub, stride;
1242   int gtid = __kmp_entry_gtid();
1243   MKLOC(loc, "GOMP_sections_start");
1244   KA_TRACE(20, ("GOMP_sections_start: T#%d\n", gtid));
1245
1246   KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
1247
1248   status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride);
1249   if (status) {
1250     KMP_DEBUG_ASSERT(stride == 1);
1251     KMP_DEBUG_ASSERT(lb > 0);
1252     KMP_ASSERT(lb == ub);
1253   } else {
1254     lb = 0;
1255   }
1256
1257   KA_TRACE(20, ("GOMP_sections_start exit: T#%d returning %u\n", gtid,
1258                 (unsigned)lb));
1259   return (unsigned)lb;
1260 }
1261
1262 unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_NEXT)(void) {
1263   int status;
1264   kmp_int lb, ub, stride;
1265   int gtid = __kmp_get_gtid();
1266   MKLOC(loc, "GOMP_sections_next");
1267   KA_TRACE(20, ("GOMP_sections_next: T#%d\n", gtid));
1268
1269 #if OMPT_SUPPORT
1270   OMPT_STORE_RETURN_ADDRESS(gtid);
1271 #endif
1272
1273   status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride);
1274   if (status) {
1275     KMP_DEBUG_ASSERT(stride == 1);
1276     KMP_DEBUG_ASSERT(lb > 0);
1277     KMP_ASSERT(lb == ub);
1278   } else {
1279     lb = 0;
1280   }
1281
1282   KA_TRACE(
1283       20, ("GOMP_sections_next exit: T#%d returning %u\n", gtid, (unsigned)lb));
1284   return (unsigned)lb;
1285 }
1286
1287 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)(
1288     void (*task)(void *), void *data, unsigned num_threads, unsigned count) {
1289   int gtid = __kmp_entry_gtid();
1290
1291 #if OMPT_SUPPORT
1292   ompt_frame_t *parent_frame;
1293
1294   if (ompt_enabled.enabled) {
1295     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);
1296     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1297     OMPT_STORE_RETURN_ADDRESS(gtid);
1298   }
1299 #endif
1300
1301   MKLOC(loc, "GOMP_parallel_sections_start");
1302   KA_TRACE(20, ("GOMP_parallel_sections_start: T#%d\n", gtid));
1303
1304   if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
1305     if (num_threads != 0) {
1306       __kmp_push_num_threads(&loc, gtid, num_threads);
1307     }
1308     __kmp_GOMP_fork_call(&loc, gtid, task,
1309                          (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9,
1310                          task, data, num_threads, &loc, kmp_nm_dynamic_chunked,
1311                          (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1);
1312   } else {
1313     __kmp_GOMP_serialized_parallel(&loc, gtid, task);
1314   }
1315
1316 #if OMPT_SUPPORT
1317   if (ompt_enabled.enabled) {
1318     parent_frame->enter_frame = ompt_data_none;
1319   }
1320 #endif
1321
1322   KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
1323
1324   KA_TRACE(20, ("GOMP_parallel_sections_start exit: T#%d\n", gtid));
1325 }
1326
1327 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END)(void) {
1328   int gtid = __kmp_get_gtid();
1329   KA_TRACE(20, ("GOMP_sections_end: T#%d\n", gtid))
1330
1331 #if OMPT_SUPPORT
1332   ompt_frame_t *ompt_frame;
1333   if (ompt_enabled.enabled) {
1334     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1335     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1336     OMPT_STORE_RETURN_ADDRESS(gtid);
1337   }
1338 #endif
1339   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
1340 #if OMPT_SUPPORT
1341   if (ompt_enabled.enabled) {
1342     ompt_frame->enter_frame = ompt_data_none;
1343   }
1344 #endif
1345
1346   KA_TRACE(20, ("GOMP_sections_end exit: T#%d\n", gtid))
1347 }
1348
1349 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT)(void) {
1350   KA_TRACE(20, ("GOMP_sections_end_nowait: T#%d\n", __kmp_get_gtid()))
1351 }
1352
1353 // libgomp has an empty function for GOMP_taskyield as of 2013-10-10
1354 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKYIELD)(void) {
1355   KA_TRACE(20, ("GOMP_taskyield: T#%d\n", __kmp_get_gtid()))
1356   return;
1357 }
1358
1359 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *),
1360                                                  void *data,
1361                                                  unsigned num_threads,
1362                                                  unsigned int flags) {
1363   int gtid = __kmp_entry_gtid();
1364   MKLOC(loc, "GOMP_parallel");
1365   KA_TRACE(20, ("GOMP_parallel: T#%d\n", gtid));
1366
1367 #if OMPT_SUPPORT
1368   ompt_task_info_t *parent_task_info, *task_info;
1369   if (ompt_enabled.enabled) {
1370     parent_task_info = __ompt_get_task_info_object(0);
1371     parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1372     OMPT_STORE_RETURN_ADDRESS(gtid);
1373   }
1374 #endif
1375   if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
1376     if (num_threads != 0) {
1377       __kmp_push_num_threads(&loc, gtid, num_threads);
1378     }
1379     if (flags != 0) {
1380       __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags);
1381     }
1382     __kmp_GOMP_fork_call(&loc, gtid, task,
1383                          (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task,
1384                          data);
1385   } else {
1386     __kmp_GOMP_serialized_parallel(&loc, gtid, task);
1387   }
1388 #if OMPT_SUPPORT
1389   if (ompt_enabled.enabled) {
1390     task_info = __ompt_get_task_info_object(0);
1391     task_info->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1392   }
1393 #endif
1394   task(data);
1395 #if OMPT_SUPPORT
1396   if (ompt_enabled.enabled) {
1397     OMPT_STORE_RETURN_ADDRESS(gtid);
1398   }
1399 #endif
1400   KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
1401 #if OMPT_SUPPORT
1402   if (ompt_enabled.enabled) {
1403     task_info->frame.exit_frame = ompt_data_none;
1404     parent_task_info->frame.enter_frame = ompt_data_none;
1405   }
1406 #endif
1407 }
1408
1409 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task)(void *),
1410                                                           void *data,
1411                                                           unsigned num_threads,
1412                                                           unsigned count,
1413                                                           unsigned flags) {
1414   int gtid = __kmp_entry_gtid();
1415   MKLOC(loc, "GOMP_parallel_sections");
1416   KA_TRACE(20, ("GOMP_parallel_sections: T#%d\n", gtid));
1417
1418 #if OMPT_SUPPORT
1419   OMPT_STORE_RETURN_ADDRESS(gtid);
1420 #endif
1421
1422   if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
1423     if (num_threads != 0) {
1424       __kmp_push_num_threads(&loc, gtid, num_threads);
1425     }
1426     if (flags != 0) {
1427       __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags);
1428     }
1429     __kmp_GOMP_fork_call(&loc, gtid, task,
1430                          (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9,
1431                          task, data, num_threads, &loc, kmp_nm_dynamic_chunked,
1432                          (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1);
1433   } else {
1434     __kmp_GOMP_serialized_parallel(&loc, gtid, task);
1435   }
1436
1437 #if OMPT_SUPPORT
1438   OMPT_STORE_RETURN_ADDRESS(gtid);
1439 #endif
1440
1441   KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
1442
1443   task(data);
1444   KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
1445   KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid));
1446 }
1447
1448 #define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post)                     \
1449   void func(void (*task)(void *), void *data, unsigned num_threads, long lb,   \
1450             long ub, long str, long chunk_sz, unsigned flags) {                \
1451     int gtid = __kmp_entry_gtid();                                             \
1452     MKLOC(loc, KMP_STR(func));                                                 \
1453     KA_TRACE(                                                                  \
1454         20,                                                                    \
1455         (KMP_STR(                                                              \
1456              func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",  \
1457          gtid, lb, ub, str, chunk_sz));                                        \
1458                                                                                \
1459     ompt_pre();                                                                \
1460     if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {                       \
1461       if (num_threads != 0) {                                                  \
1462         __kmp_push_num_threads(&loc, gtid, num_threads);                       \
1463       }                                                                        \
1464       if (flags != 0) {                                                        \
1465         __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags);              \
1466       }                                                                        \
1467       __kmp_GOMP_fork_call(&loc, gtid, task,                                   \
1468                            (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \
1469                            9, task, data, num_threads, &loc, (schedule), lb,   \
1470                            (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz);    \
1471     } else {                                                                   \
1472       __kmp_GOMP_serialized_parallel(&loc, gtid, task);                        \
1473     }                                                                          \
1474                                                                                \
1475     IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
1476     KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                              \
1477                       (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,          \
1478                       (schedule) != kmp_sch_static);                           \
1479     task(data);                                                                \
1480     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();                         \
1481     ompt_post();                                                               \
1482                                                                                \
1483     KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid));                       \
1484   }
1485
1486 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC),
1487               kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1488 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC),
1489               kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1490 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED),
1491               kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1492 PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME),
1493               kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
1494
1495 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) {
1496   int gtid = __kmp_entry_gtid();
1497   MKLOC(loc, "GOMP_taskgroup_start");
1498   KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid));
1499
1500 #if OMPT_SUPPORT
1501   if (ompt_enabled.enabled)
1502     OMPT_STORE_RETURN_ADDRESS(gtid);
1503 #endif
1504
1505   __kmpc_taskgroup(&loc, gtid);
1506
1507   return;
1508 }
1509
1510 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_END)(void) {
1511   int gtid = __kmp_get_gtid();
1512   MKLOC(loc, "GOMP_taskgroup_end");
1513   KA_TRACE(20, ("GOMP_taskgroup_end: T#%d\n", gtid));
1514
1515 #if OMPT_SUPPORT
1516   if (ompt_enabled.enabled)
1517     OMPT_STORE_RETURN_ADDRESS(gtid);
1518 #endif
1519
1520   __kmpc_end_taskgroup(&loc, gtid);
1521
1522   return;
1523 }
1524
1525 static kmp_int32 __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) {
1526   kmp_int32 cncl_kind = 0;
1527   switch (gomp_kind) {
1528   case 1:
1529     cncl_kind = cancel_parallel;
1530     break;
1531   case 2:
1532     cncl_kind = cancel_loop;
1533     break;
1534   case 4:
1535     cncl_kind = cancel_sections;
1536     break;
1537   case 8:
1538     cncl_kind = cancel_taskgroup;
1539     break;
1540   }
1541   return cncl_kind;
1542 }
1543
1544 // Return true if cancellation should take place, false otherwise
1545 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which) {
1546   int gtid = __kmp_get_gtid();
1547   MKLOC(loc, "GOMP_cancellation_point");
1548   KA_TRACE(20, ("GOMP_cancellation_point: T#%d which:%d\n", gtid, which));
1549   kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
1550   return __kmpc_cancellationpoint(&loc, gtid, cncl_kind);
1551 }
1552
1553 // Return true if cancellation should take place, false otherwise
1554 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel) {
1555   int gtid = __kmp_get_gtid();
1556   MKLOC(loc, "GOMP_cancel");
1557   KA_TRACE(20, ("GOMP_cancel: T#%d which:%d do_cancel:%d\n", gtid, which,
1558                 (int)do_cancel));
1559   kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
1560
1561   if (do_cancel == FALSE) {
1562     return __kmpc_cancellationpoint(&loc, gtid, cncl_kind);
1563   } else {
1564     return __kmpc_cancel(&loc, gtid, cncl_kind);
1565   }
1566 }
1567
1568 // Return true if cancellation should take place, false otherwise
1569 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void) {
1570   int gtid = __kmp_get_gtid();
1571   KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid));
1572   return __kmp_barrier_gomp_cancel(gtid);
1573 }
1574
1575 // Return true if cancellation should take place, false otherwise
1576 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void) {
1577   int gtid = __kmp_get_gtid();
1578   KA_TRACE(20, ("GOMP_sections_end_cancel: T#%d\n", gtid));
1579   return __kmp_barrier_gomp_cancel(gtid);
1580 }
1581
1582 // Return true if cancellation should take place, false otherwise
1583 bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_CANCEL)(void) {
1584   int gtid = __kmp_get_gtid();
1585   KA_TRACE(20, ("GOMP_loop_end_cancel: T#%d\n", gtid));
1586   return __kmp_barrier_gomp_cancel(gtid);
1587 }
1588
1589 // All target functions are empty as of 2014-05-29
1590 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET)(int device, void (*fn)(void *),
1591                                                const void *openmp_target,
1592                                                size_t mapnum, void **hostaddrs,
1593                                                size_t *sizes,
1594                                                unsigned char *kinds) {
1595   return;
1596 }
1597
1598 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_DATA)(
1599     int device, const void *openmp_target, size_t mapnum, void **hostaddrs,
1600     size_t *sizes, unsigned char *kinds) {
1601   return;
1602 }
1603
1604 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_END_DATA)(void) { return; }
1605
1606 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_UPDATE)(
1607     int device, const void *openmp_target, size_t mapnum, void **hostaddrs,
1608     size_t *sizes, unsigned char *kinds) {
1609   return;
1610 }
1611
1612 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS)(unsigned int num_teams,
1613                                               unsigned int thread_limit) {
1614   return;
1615 }
1616
1617 // Task duplication function which copies src to dest (both are
1618 // preallocated task structures)
1619 static void __kmp_gomp_task_dup(kmp_task_t *dest, kmp_task_t *src,
1620                                 kmp_int32 last_private) {
1621   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(src);
1622   if (taskdata->td_copy_func) {
1623     (taskdata->td_copy_func)(dest->shareds, src->shareds);
1624   }
1625 }
1626
1627 #ifdef __cplusplus
1628 } // extern "C"
1629 #endif
1630
1631 template <typename T>
1632 void __GOMP_taskloop(void (*func)(void *), void *data,
1633                      void (*copy_func)(void *, void *), long arg_size,
1634                      long arg_align, unsigned gomp_flags,
1635                      unsigned long num_tasks, int priority, T start, T end,
1636                      T step) {
1637   typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
1638   MKLOC(loc, "GOMP_taskloop");
1639   int sched;
1640   T *loop_bounds;
1641   int gtid = __kmp_entry_gtid();
1642   kmp_int32 flags = 0;
1643   int if_val = gomp_flags & (1u << 10);
1644   int nogroup = gomp_flags & (1u << 11);
1645   int up = gomp_flags & (1u << 8);
1646   p_task_dup_t task_dup = NULL;
1647   kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1648 #ifdef KMP_DEBUG
1649   {
1650     char *buff;
1651     buff = __kmp_str_format(
1652         "GOMP_taskloop: T#%%d: func:%%p data:%%p copy_func:%%p "
1653         "arg_size:%%ld arg_align:%%ld gomp_flags:0x%%x num_tasks:%%lu "
1654         "priority:%%d start:%%%s end:%%%s step:%%%s\n",
1655         traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec);
1656     KA_TRACE(20, (buff, gtid, func, data, copy_func, arg_size, arg_align,
1657                   gomp_flags, num_tasks, priority, start, end, step));
1658     __kmp_str_free(&buff);
1659   }
1660 #endif
1661   KMP_ASSERT((size_t)arg_size >= 2 * sizeof(T));
1662   KMP_ASSERT(arg_align > 0);
1663   // The low-order bit is the "untied" flag
1664   if (!(gomp_flags & 1)) {
1665     input_flags->tiedness = 1;
1666   }
1667   // The second low-order bit is the "final" flag
1668   if (gomp_flags & 2) {
1669     input_flags->final = 1;
1670   }
1671   // Negative step flag
1672   if (!up) {
1673     // If step is flagged as negative, but isn't properly sign extended
1674     // Then manually sign extend it.  Could be a short, int, char embedded
1675     // in a long.  So cannot assume any cast.
1676     if (step > 0) {
1677       for (int i = sizeof(T) * CHAR_BIT - 1; i >= 0L; --i) {
1678         // break at the first 1 bit
1679         if (step & ((T)1 << i))
1680           break;
1681         step |= ((T)1 << i);
1682       }
1683     }
1684   }
1685   input_flags->native = 1;
1686   // Figure out if none/grainsize/num_tasks clause specified
1687   if (num_tasks > 0) {
1688     if (gomp_flags & (1u << 9))
1689       sched = 1; // grainsize specified
1690     else
1691       sched = 2; // num_tasks specified
1692     // neither grainsize nor num_tasks specified
1693   } else {
1694     sched = 0;
1695   }
1696
1697   // __kmp_task_alloc() sets up all other flags
1698   kmp_task_t *task =
1699       __kmp_task_alloc(&loc, gtid, input_flags, sizeof(kmp_task_t),
1700                        arg_size + arg_align - 1, (kmp_routine_entry_t)func);
1701   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1702   taskdata->td_copy_func = copy_func;
1703   taskdata->td_size_loop_bounds = sizeof(T);
1704
1705   // re-align shareds if needed and setup firstprivate copy constructors
1706   // through the task_dup mechanism
1707   task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) /
1708                            arg_align * arg_align);
1709   if (copy_func) {
1710     task_dup = __kmp_gomp_task_dup;
1711   }
1712   KMP_MEMCPY(task->shareds, data, arg_size);
1713
1714   loop_bounds = (T *)task->shareds;
1715   loop_bounds[0] = start;
1716   loop_bounds[1] = end + (up ? -1 : 1);
1717   __kmpc_taskloop(&loc, gtid, task, if_val, (kmp_uint64 *)&(loop_bounds[0]),
1718                   (kmp_uint64 *)&(loop_bounds[1]), (kmp_int64)step, nogroup,
1719                   sched, (kmp_uint64)num_tasks, (void *)task_dup);
1720 }
1721
1722 // 4 byte version of GOMP_doacross_post
1723 // This verison needs to create a temporary array which converts 4 byte
1724 // integers into 8 byte integeres
1725 template <typename T, bool need_conversion = (sizeof(long) == 4)>
1726 void __kmp_GOMP_doacross_post(T *count);
1727
1728 template <> void __kmp_GOMP_doacross_post<long, true>(long *count) {
1729   int gtid = __kmp_entry_gtid();
1730   kmp_info_t *th = __kmp_threads[gtid];
1731   MKLOC(loc, "GOMP_doacross_post");
1732   kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0];
1733   kmp_int64 *vec =
1734       (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims);
1735   for (kmp_int64 i = 0; i < num_dims; ++i) {
1736     vec[i] = (kmp_int64)count[i];
1737   }
1738   __kmpc_doacross_post(&loc, gtid, vec);
1739   __kmp_thread_free(th, vec);
1740 }
1741
1742 // 8 byte versions of GOMP_doacross_post
1743 // This version can just pass in the count array directly instead of creating
1744 // a temporary array
1745 template <> void __kmp_GOMP_doacross_post<long, false>(long *count) {
1746   int gtid = __kmp_entry_gtid();
1747   MKLOC(loc, "GOMP_doacross_post");
1748   __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count));
1749 }
1750
1751 template <typename T> void __kmp_GOMP_doacross_wait(T first, va_list args) {
1752   int gtid = __kmp_entry_gtid();
1753   kmp_info_t *th = __kmp_threads[gtid];
1754   MKLOC(loc, "GOMP_doacross_wait");
1755   kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0];
1756   kmp_int64 *vec =
1757       (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims);
1758   vec[0] = (kmp_int64)first;
1759   for (kmp_int64 i = 1; i < num_dims; ++i) {
1760     T item = va_arg(args, T);
1761     vec[i] = (kmp_int64)item;
1762   }
1763   __kmpc_doacross_wait(&loc, gtid, vec);
1764   __kmp_thread_free(th, vec);
1765   return;
1766 }
1767
1768 #ifdef __cplusplus
1769 extern "C" {
1770 #endif // __cplusplus
1771
1772 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP)(
1773     void (*func)(void *), void *data, void (*copy_func)(void *, void *),
1774     long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks,
1775     int priority, long start, long end, long step) {
1776   __GOMP_taskloop<long>(func, data, copy_func, arg_size, arg_align, gomp_flags,
1777                         num_tasks, priority, start, end, step);
1778 }
1779
1780 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP_ULL)(
1781     void (*func)(void *), void *data, void (*copy_func)(void *, void *),
1782     long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks,
1783     int priority, unsigned long long start, unsigned long long end,
1784     unsigned long long step) {
1785   __GOMP_taskloop<unsigned long long>(func, data, copy_func, arg_size,
1786                                       arg_align, gomp_flags, num_tasks,
1787                                       priority, start, end, step);
1788 }
1789
1790 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_POST)(long *count) {
1791   __kmp_GOMP_doacross_post(count);
1792 }
1793
1794 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_WAIT)(long first, ...) {
1795   va_list args;
1796   va_start(args, first);
1797   __kmp_GOMP_doacross_wait<long>(first, args);
1798   va_end(args);
1799 }
1800
1801 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_POST)(
1802     unsigned long long *count) {
1803   int gtid = __kmp_entry_gtid();
1804   MKLOC(loc, "GOMP_doacross_ull_post");
1805   __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count));
1806 }
1807
1808 void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT)(
1809     unsigned long long first, ...) {
1810   va_list args;
1811   va_start(args, first);
1812   __kmp_GOMP_doacross_wait<unsigned long long>(first, args);
1813   va_end(args);
1814 }
1815
1816 /* The following sections of code create aliases for the GOMP_* functions, then
1817    create versioned symbols using the assembler directive .symver. This is only
1818    pertinent for ELF .so library. The KMP_VERSION_SYMBOL macro is defined in
1819    kmp_os.h  */
1820
1821 #ifdef KMP_USE_VERSION_SYMBOLS
1822 // GOMP_1.0 versioned symbols
1823 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_END, 10, "GOMP_1.0");
1824 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_START, 10, "GOMP_1.0");
1825 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER, 10, "GOMP_1.0");
1826 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_END, 10, "GOMP_1.0");
1827 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_END, 10, "GOMP_1.0");
1828 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_START, 10, "GOMP_1.0");
1829 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_START, 10, "GOMP_1.0");
1830 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT, 10, "GOMP_1.0");
1831 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START, 10, "GOMP_1.0");
1832 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END, 10, "GOMP_1.0");
1833 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_NOWAIT, 10, "GOMP_1.0");
1834 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT, 10, "GOMP_1.0");
1835 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_START, 10, "GOMP_1.0");
1836 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT, 10, "GOMP_1.0");
1837 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START, 10,
1838                    "GOMP_1.0");
1839 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT, 10, "GOMP_1.0");
1840 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START, 10, "GOMP_1.0");
1841 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT, 10, "GOMP_1.0");
1842 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START, 10,
1843                    "GOMP_1.0");
1844 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT, 10, "GOMP_1.0");
1845 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START, 10, "GOMP_1.0");
1846 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT, 10, "GOMP_1.0");
1847 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_START, 10, "GOMP_1.0");
1848 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT, 10, "GOMP_1.0");
1849 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_START, 10, "GOMP_1.0");
1850 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_END, 10, "GOMP_1.0");
1851 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_START, 10, "GOMP_1.0");
1852 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_END, 10, "GOMP_1.0");
1853 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START, 10,
1854                    "GOMP_1.0");
1855 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START, 10,
1856                    "GOMP_1.0");
1857 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START, 10,
1858                    "GOMP_1.0");
1859 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START, 10,
1860                    "GOMP_1.0");
1861 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START, 10, "GOMP_1.0");
1862 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_START, 10, "GOMP_1.0");
1863 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END, 10, "GOMP_1.0");
1864 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT, 10, "GOMP_1.0");
1865 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_NEXT, 10, "GOMP_1.0");
1866 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_START, 10, "GOMP_1.0");
1867 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_END, 10, "GOMP_1.0");
1868 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_START, 10, "GOMP_1.0");
1869 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_START, 10, "GOMP_1.0");
1870
1871 // GOMP_2.0 versioned symbols
1872 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASK, 20, "GOMP_2.0");
1873 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKWAIT, 20, "GOMP_2.0");
1874 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT, 20, "GOMP_2.0");
1875 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START, 20, "GOMP_2.0");
1876 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT, 20, "GOMP_2.0");
1877 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START, 20, "GOMP_2.0");
1878 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT, 20,
1879                    "GOMP_2.0");
1880 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START, 20,
1881                    "GOMP_2.0");
1882 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT, 20,
1883                    "GOMP_2.0");
1884 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START, 20,
1885                    "GOMP_2.0");
1886 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, 20,
1887                    "GOMP_2.0");
1888 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START, 20,
1889                    "GOMP_2.0");
1890 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT, 20,
1891                    "GOMP_2.0");
1892 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START, 20,
1893                    "GOMP_2.0");
1894 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT, 20, "GOMP_2.0");
1895 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START, 20, "GOMP_2.0");
1896 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT, 20, "GOMP_2.0");
1897 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START, 20, "GOMP_2.0");
1898
1899 // GOMP_3.0 versioned symbols
1900 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKYIELD, 30, "GOMP_3.0");
1901
1902 // GOMP_4.0 versioned symbols
1903 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL, 40, "GOMP_4.0");
1904 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS, 40, "GOMP_4.0");
1905 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC, 40, "GOMP_4.0");
1906 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED, 40, "GOMP_4.0");
1907 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME, 40, "GOMP_4.0");
1908 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC, 40, "GOMP_4.0");
1909 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_START, 40, "GOMP_4.0");
1910 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_END, 40, "GOMP_4.0");
1911 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER_CANCEL, 40, "GOMP_4.0");
1912 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCEL, 40, "GOMP_4.0");
1913 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCELLATION_POINT, 40, "GOMP_4.0");
1914 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_CANCEL, 40, "GOMP_4.0");
1915 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL, 40, "GOMP_4.0");
1916 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET, 40, "GOMP_4.0");
1917 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_DATA, 40, "GOMP_4.0");
1918 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_END_DATA, 40, "GOMP_4.0");
1919 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_UPDATE, 40, "GOMP_4.0");
1920 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TEAMS, 40, "GOMP_4.0");
1921
1922 // GOMP_4.5 versioned symbols
1923 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP, 45, "GOMP_4.5");
1924 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP_ULL, 45, "GOMP_4.5");
1925 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_POST, 45, "GOMP_4.5");
1926 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_WAIT, 45, "GOMP_4.5");
1927 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START, 45,
1928                    "GOMP_4.5");
1929 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START, 45,
1930                    "GOMP_4.5");
1931 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START, 45,
1932                    "GOMP_4.5");
1933 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START, 45,
1934                    "GOMP_4.5");
1935 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_POST, 45, "GOMP_4.5");
1936 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT, 45, "GOMP_4.5");
1937 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START, 45,
1938                    "GOMP_4.5");
1939 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START, 45,
1940                    "GOMP_4.5");
1941 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START, 45,
1942                    "GOMP_4.5");
1943 KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START, 45,
1944                    "GOMP_4.5");
1945
1946 #endif // KMP_USE_VERSION_SYMBOLS
1947
1948 #ifdef __cplusplus
1949 } // extern "C"
1950 #endif // __cplusplus