14 #include "kmp_affinity.h" 15 #include "kmp_atomic.h" 16 #include "kmp_environment.h" 17 #include "kmp_error.h" 21 #include "kmp_settings.h" 22 #include "kmp_stats.h" 24 #include "kmp_wait_release.h" 25 #include "kmp_wrapper_getpid.h" 26 #include "kmp_dispatch.h" 27 #if KMP_USE_HIER_SCHED 28 #include "kmp_dispatch_hier.h" 32 #include "ompt-specific.h" 35 #if OMP_PROFILING_SUPPORT 36 #include "llvm/Support/TimeProfiler.h" 37 static char *ProfileTraceFile =
nullptr;
41 #define KMP_USE_PRCTL 0 47 #include "tsan_annotations.h" 58 #if defined(KMP_GOMP_COMPAT) 59 char const __kmp_version_alt_comp[] =
60 KMP_VERSION_PREFIX
"alternative compiler support: yes";
63 char const __kmp_version_omp_api[] =
64 KMP_VERSION_PREFIX
"API version: 5.0 (201611)";
67 char const __kmp_version_lock[] =
68 KMP_VERSION_PREFIX
"lock type: run time selectable";
71 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y)) 76 kmp_info_t __kmp_monitor;
81 void __kmp_cleanup(
void);
83 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *,
int tid,
85 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
86 kmp_internal_control_t *new_icvs,
88 #if KMP_AFFINITY_SUPPORTED 89 static void __kmp_partition_places(kmp_team_t *team,
90 int update_master_only = 0);
92 static void __kmp_do_serial_initialize(
void);
93 void __kmp_fork_barrier(
int gtid,
int tid);
94 void __kmp_join_barrier(
int gtid);
95 void __kmp_setup_icv_copy(kmp_team_t *team,
int new_nproc,
96 kmp_internal_control_t *new_icvs,
ident_t *loc);
98 #ifdef USE_LOAD_BALANCE 99 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc);
102 static int __kmp_expand_threads(
int nNeed);
104 static int __kmp_unregister_root_other_thread(
int gtid);
106 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root);
107 kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
112 int __kmp_get_global_thread_id() {
114 kmp_info_t **other_threads;
122 (
"*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
123 __kmp_nth, __kmp_all_nth));
130 if (!TCR_4(__kmp_init_gtid))
133 #ifdef KMP_TDATA_GTID 134 if (TCR_4(__kmp_gtid_mode) >= 3) {
135 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using TDATA\n"));
139 if (TCR_4(__kmp_gtid_mode) >= 2) {
140 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using keyed TLS\n"));
141 return __kmp_gtid_get_specific();
143 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using internal alg.\n"));
145 stack_addr = (
char *)&stack_data;
146 other_threads = __kmp_threads;
159 for (i = 0; i < __kmp_threads_capacity; i++) {
161 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
165 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
166 stack_base = (
char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
170 if (stack_addr <= stack_base) {
171 size_t stack_diff = stack_base - stack_addr;
173 if (stack_diff <= stack_size) {
176 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
184 (
"*** __kmp_get_global_thread_id: internal alg. failed to find " 185 "thread, using TLS\n"));
186 i = __kmp_gtid_get_specific();
196 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
197 KMP_FATAL(StackOverflow, i);
200 stack_base = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
201 if (stack_addr > stack_base) {
202 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
204 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
207 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
208 stack_base - stack_addr);
212 if (__kmp_storage_map) {
213 char *stack_end = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
214 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
215 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
216 other_threads[i]->th.th_info.ds.ds_stacksize,
217 "th_%d stack (refinement)", i);
222 int __kmp_get_global_thread_id_reg() {
225 if (!__kmp_init_serial) {
228 #ifdef KMP_TDATA_GTID 229 if (TCR_4(__kmp_gtid_mode) >= 3) {
230 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using TDATA\n"));
234 if (TCR_4(__kmp_gtid_mode) >= 2) {
235 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
236 gtid = __kmp_gtid_get_specific();
239 (
"*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
240 gtid = __kmp_get_global_thread_id();
244 if (gtid == KMP_GTID_DNE) {
246 (
"__kmp_get_global_thread_id_reg: Encountered new root thread. " 247 "Registering a new gtid.\n"));
248 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
249 if (!__kmp_init_serial) {
250 __kmp_do_serial_initialize();
251 gtid = __kmp_gtid_get_specific();
253 gtid = __kmp_register_root(FALSE);
255 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
259 KMP_DEBUG_ASSERT(gtid >= 0);
265 void __kmp_check_stack_overlap(kmp_info_t *th) {
267 char *stack_beg = NULL;
268 char *stack_end = NULL;
271 KA_TRACE(10, (
"__kmp_check_stack_overlap: called\n"));
272 if (__kmp_storage_map) {
273 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
274 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
276 gtid = __kmp_gtid_from_thread(th);
278 if (gtid == KMP_GTID_MONITOR) {
279 __kmp_print_storage_map_gtid(
280 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
281 "th_%s stack (%s)",
"mon",
282 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
284 __kmp_print_storage_map_gtid(
285 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
286 "th_%d stack (%s)", gtid,
287 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
293 gtid = __kmp_gtid_from_thread(th);
294 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
296 (
"__kmp_check_stack_overlap: performing extensive checking\n"));
297 if (stack_beg == NULL) {
298 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
299 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
302 for (f = 0; f < __kmp_threads_capacity; f++) {
303 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
305 if (f_th && f_th != th) {
306 char *other_stack_end =
307 (
char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
308 char *other_stack_beg =
309 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
310 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
311 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
314 if (__kmp_storage_map)
315 __kmp_print_storage_map_gtid(
316 -1, other_stack_beg, other_stack_end,
317 (
size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
318 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
320 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
326 KA_TRACE(10, (
"__kmp_check_stack_overlap: returning\n"));
331 void __kmp_infinite_loop(
void) {
332 static int done = FALSE;
339 #define MAX_MESSAGE 512 341 void __kmp_print_storage_map_gtid(
int gtid,
void *p1,
void *p2,
size_t size,
342 char const *format, ...) {
343 char buffer[MAX_MESSAGE];
346 va_start(ap, format);
347 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP storage map: %p %p%8lu %s\n", p1,
348 p2, (
unsigned long)size, format);
349 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
350 __kmp_vprintf(kmp_err, buffer, ap);
351 #if KMP_PRINT_DATA_PLACEMENT 354 if (p1 <= p2 && (
char *)p2 - (
char *)p1 == size) {
355 if (__kmp_storage_map_verbose) {
356 node = __kmp_get_host_node(p1);
358 __kmp_storage_map_verbose = FALSE;
362 int localProc = __kmp_get_cpu_from_gtid(gtid);
364 const int page_size = KMP_GET_PAGE_SIZE();
366 p1 = (
void *)((
size_t)p1 & ~((size_t)page_size - 1));
367 p2 = (
void *)(((
size_t)p2 - 1) & ~((
size_t)page_size - 1));
369 __kmp_printf_no_lock(
" GTID %d localNode %d\n", gtid,
372 __kmp_printf_no_lock(
" GTID %d\n", gtid);
381 (
char *)p1 += page_size;
382 }
while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
383 __kmp_printf_no_lock(
" %p-%p memNode %d\n", last, (
char *)p1 - 1,
387 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p1,
388 (
char *)p1 + (page_size - 1),
389 __kmp_get_host_node(p1));
391 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p2,
392 (
char *)p2 + (page_size - 1),
393 __kmp_get_host_node(p2));
399 __kmp_printf_no_lock(
" %s\n", KMP_I18N_STR(StorageMapWarning));
402 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
405 void __kmp_warn(
char const *format, ...) {
406 char buffer[MAX_MESSAGE];
409 if (__kmp_generate_warnings == kmp_warnings_off) {
413 va_start(ap, format);
415 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP warning: %s\n", format);
416 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
417 __kmp_vprintf(kmp_err, buffer, ap);
418 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
423 void __kmp_abort_process() {
425 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
427 if (__kmp_debug_buf) {
428 __kmp_dump_debug_buffer();
431 if (KMP_OS_WINDOWS) {
434 __kmp_global.g.g_abort = SIGABRT;
448 __kmp_unregister_library();
452 __kmp_infinite_loop();
453 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
457 void __kmp_abort_thread(
void) {
460 __kmp_infinite_loop();
466 static void __kmp_print_thread_storage_map(kmp_info_t *thr,
int gtid) {
467 __kmp_print_storage_map_gtid(gtid, thr, thr + 1,
sizeof(kmp_info_t),
"th_%d",
470 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
471 sizeof(kmp_desc_t),
"th_%d.th_info", gtid);
473 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
474 sizeof(kmp_local_t),
"th_%d.th_local", gtid);
476 __kmp_print_storage_map_gtid(
477 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
478 sizeof(kmp_balign_t) * bs_last_barrier,
"th_%d.th_bar", gtid);
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
481 &thr->th.th_bar[bs_plain_barrier + 1],
482 sizeof(kmp_balign_t),
"th_%d.th_bar[plain]",
485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
486 &thr->th.th_bar[bs_forkjoin_barrier + 1],
487 sizeof(kmp_balign_t),
"th_%d.th_bar[forkjoin]",
490 #if KMP_FAST_REDUCTION_BARRIER 491 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
492 &thr->th.th_bar[bs_reduction_barrier + 1],
493 sizeof(kmp_balign_t),
"th_%d.th_bar[reduction]",
495 #endif // KMP_FAST_REDUCTION_BARRIER 501 static void __kmp_print_team_storage_map(
const char *header, kmp_team_t *team,
502 int team_id,
int num_thr) {
503 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
504 __kmp_print_storage_map_gtid(-1, team, team + 1,
sizeof(kmp_team_t),
"%s_%d",
507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
508 &team->t.t_bar[bs_last_barrier],
509 sizeof(kmp_balign_team_t) * bs_last_barrier,
510 "%s_%d.t_bar", header, team_id);
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
513 &team->t.t_bar[bs_plain_barrier + 1],
514 sizeof(kmp_balign_team_t),
"%s_%d.t_bar[plain]",
517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
518 &team->t.t_bar[bs_forkjoin_barrier + 1],
519 sizeof(kmp_balign_team_t),
520 "%s_%d.t_bar[forkjoin]", header, team_id);
522 #if KMP_FAST_REDUCTION_BARRIER 523 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
524 &team->t.t_bar[bs_reduction_barrier + 1],
525 sizeof(kmp_balign_team_t),
526 "%s_%d.t_bar[reduction]", header, team_id);
527 #endif // KMP_FAST_REDUCTION_BARRIER 529 __kmp_print_storage_map_gtid(
530 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
531 sizeof(kmp_disp_t) * num_thr,
"%s_%d.t_dispatch", header, team_id);
533 __kmp_print_storage_map_gtid(
534 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
535 sizeof(kmp_info_t *) * num_thr,
"%s_%d.t_threads", header, team_id);
537 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
538 &team->t.t_disp_buffer[num_disp_buff],
539 sizeof(dispatch_shared_info_t) * num_disp_buff,
540 "%s_%d.t_disp_buffer", header, team_id);
543 static void __kmp_init_allocator() {
544 __kmp_init_memkind();
545 __kmp_init_target_mem();
547 static void __kmp_fini_allocator() { __kmp_fini_memkind(); }
554 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
559 case DLL_PROCESS_ATTACH:
560 KA_TRACE(10, (
"DllMain: PROCESS_ATTACH\n"));
564 case DLL_PROCESS_DETACH:
565 KA_TRACE(10, (
"DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
578 if (lpReserved == NULL)
579 __kmp_internal_end_library(__kmp_gtid_get_specific());
583 case DLL_THREAD_ATTACH:
584 KA_TRACE(10, (
"DllMain: THREAD_ATTACH\n"));
590 case DLL_THREAD_DETACH:
591 KA_TRACE(10, (
"DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
593 __kmp_internal_end_thread(__kmp_gtid_get_specific());
604 void __kmp_parallel_deo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
605 int gtid = *gtid_ref;
606 #ifdef BUILD_PARALLEL_ORDERED 607 kmp_team_t *team = __kmp_team_from_gtid(gtid);
610 if (__kmp_env_consistency_check) {
611 if (__kmp_threads[gtid]->th.th_root->r.r_active)
612 #if KMP_USE_DYNAMIC_LOCK 613 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
615 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
618 #ifdef BUILD_PARALLEL_ORDERED 619 if (!team->t.t_serialized) {
621 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,
629 void __kmp_parallel_dxo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
630 int gtid = *gtid_ref;
631 #ifdef BUILD_PARALLEL_ORDERED 632 int tid = __kmp_tid_from_gtid(gtid);
633 kmp_team_t *team = __kmp_team_from_gtid(gtid);
636 if (__kmp_env_consistency_check) {
637 if (__kmp_threads[gtid]->th.th_root->r.r_active)
638 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
640 #ifdef BUILD_PARALLEL_ORDERED 641 if (!team->t.t_serialized) {
646 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
656 int __kmp_enter_single(
int gtid,
ident_t *id_ref,
int push_ws) {
661 if (!TCR_4(__kmp_init_parallel))
662 __kmp_parallel_initialize();
663 __kmp_resume_if_soft_paused();
665 th = __kmp_threads[gtid];
666 team = th->th.th_team;
669 th->th.th_ident = id_ref;
671 if (team->t.t_serialized) {
674 kmp_int32 old_this = th->th.th_local.this_construct;
676 ++th->th.th_local.this_construct;
680 if (team->t.t_construct == old_this) {
681 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
682 th->th.th_local.this_construct);
685 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
686 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL &&
687 team->t.t_active_level == 1) {
689 __kmp_itt_metadata_single(id_ref);
694 if (__kmp_env_consistency_check) {
695 if (status && push_ws) {
696 __kmp_push_workshare(gtid, ct_psingle, id_ref);
698 __kmp_check_workshare(gtid, ct_psingle, id_ref);
703 __kmp_itt_single_start(gtid);
709 void __kmp_exit_single(
int gtid) {
711 __kmp_itt_single_end(gtid);
713 if (__kmp_env_consistency_check)
714 __kmp_pop_workshare(gtid, ct_psingle, NULL);
723 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
724 int master_tid,
int set_nthreads,
728 KMP_DEBUG_ASSERT(__kmp_init_serial);
729 KMP_DEBUG_ASSERT(root && parent_team);
730 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
734 new_nthreads = set_nthreads;
735 if (!get__dynamic_2(parent_team, master_tid)) {
738 #ifdef USE_LOAD_BALANCE 739 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
740 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
741 if (new_nthreads == 1) {
742 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced " 743 "reservation to 1 thread\n",
747 if (new_nthreads < set_nthreads) {
748 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced " 749 "reservation to %d threads\n",
750 master_tid, new_nthreads));
754 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
755 new_nthreads = __kmp_avail_proc - __kmp_nth +
756 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
757 if (new_nthreads <= 1) {
758 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced " 759 "reservation to 1 thread\n",
763 if (new_nthreads < set_nthreads) {
764 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced " 765 "reservation to %d threads\n",
766 master_tid, new_nthreads));
768 new_nthreads = set_nthreads;
770 }
else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
771 if (set_nthreads > 2) {
772 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
773 new_nthreads = (new_nthreads % set_nthreads) + 1;
774 if (new_nthreads == 1) {
775 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced " 776 "reservation to 1 thread\n",
780 if (new_nthreads < set_nthreads) {
781 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced " 782 "reservation to %d threads\n",
783 master_tid, new_nthreads));
791 if (__kmp_nth + new_nthreads -
792 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
794 int tl_nthreads = __kmp_max_nth - __kmp_nth +
795 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
796 if (tl_nthreads <= 0) {
801 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
802 __kmp_reserve_warn = 1;
803 __kmp_msg(kmp_ms_warning,
804 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
805 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
807 if (tl_nthreads == 1) {
808 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT " 809 "reduced reservation to 1 thread\n",
813 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced " 814 "reservation to %d threads\n",
815 master_tid, tl_nthreads));
816 new_nthreads = tl_nthreads;
820 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
821 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
822 if (cg_nthreads + new_nthreads -
823 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
825 int tl_nthreads = max_cg_threads - cg_nthreads +
826 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
827 if (tl_nthreads <= 0) {
832 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
833 __kmp_reserve_warn = 1;
834 __kmp_msg(kmp_ms_warning,
835 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
836 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
838 if (tl_nthreads == 1) {
839 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT " 840 "reduced reservation to 1 thread\n",
844 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced " 845 "reservation to %d threads\n",
846 master_tid, tl_nthreads));
847 new_nthreads = tl_nthreads;
853 capacity = __kmp_threads_capacity;
854 if (TCR_PTR(__kmp_threads[0]) == NULL) {
860 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
861 capacity -= __kmp_hidden_helper_threads_num;
863 if (__kmp_nth + new_nthreads -
864 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
867 int slotsRequired = __kmp_nth + new_nthreads -
868 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
870 int slotsAdded = __kmp_expand_threads(slotsRequired);
871 if (slotsAdded < slotsRequired) {
873 new_nthreads -= (slotsRequired - slotsAdded);
874 KMP_ASSERT(new_nthreads >= 1);
877 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
878 __kmp_reserve_warn = 1;
879 if (__kmp_tp_cached) {
880 __kmp_msg(kmp_ms_warning,
881 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
882 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
883 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
885 __kmp_msg(kmp_ms_warning,
886 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
887 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
894 if (new_nthreads == 1) {
896 (
"__kmp_reserve_threads: T#%d serializing team after reclaiming " 897 "dead roots and rechecking; requested %d threads\n",
898 __kmp_get_gtid(), set_nthreads));
900 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d allocating %d threads; requested" 902 __kmp_get_gtid(), new_nthreads, set_nthreads));
911 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
912 kmp_info_t *master_th,
int master_gtid) {
916 KA_TRACE(10, (
"__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
917 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
921 master_th->th.th_info.ds.ds_tid = 0;
922 master_th->th.th_team = team;
923 master_th->th.th_team_nproc = team->t.t_nproc;
924 master_th->th.th_team_master = master_th;
925 master_th->th.th_team_serialized = FALSE;
926 master_th->th.th_dispatch = &team->t.t_dispatch[0];
929 #if KMP_NESTED_HOT_TEAMS 931 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
934 int level = team->t.t_active_level - 1;
935 if (master_th->th.th_teams_microtask) {
936 if (master_th->th.th_teams_size.nteams > 1) {
940 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
941 master_th->th.th_teams_level == team->t.t_level) {
946 if (level < __kmp_hot_teams_max_level) {
947 if (hot_teams[level].hot_team) {
949 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
953 hot_teams[level].hot_team = team;
954 hot_teams[level].hot_team_nth = team->t.t_nproc;
961 use_hot_team = team == root->r.r_hot_team;
966 team->t.t_threads[0] = master_th;
967 __kmp_initialize_info(master_th, team, 0, master_gtid);
970 for (i = 1; i < team->t.t_nproc; i++) {
973 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
974 team->t.t_threads[i] = thr;
975 KMP_DEBUG_ASSERT(thr);
976 KMP_DEBUG_ASSERT(thr->th.th_team == team);
978 KA_TRACE(20, (
"__kmp_fork_team_threads: T#%d(%d:%d) init arrived " 979 "T#%d(%d:%d) join =%llu, plain=%llu\n",
980 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
981 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
982 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
983 team->t.t_bar[bs_plain_barrier].b_arrived));
984 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
985 thr->th.th_teams_level = master_th->th.th_teams_level;
986 thr->th.th_teams_size = master_th->th.th_teams_size;
989 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
990 for (b = 0; b < bs_last_barrier; ++b) {
991 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
992 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
994 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1000 #if KMP_AFFINITY_SUPPORTED 1001 __kmp_partition_places(team);
1005 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1006 for (i = 0; i < team->t.t_nproc; i++) {
1007 kmp_info_t *thr = team->t.t_threads[i];
1008 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1009 thr->th.th_prev_level != team->t.t_level) {
1010 team->t.t_display_affinity = 1;
1019 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1023 inline static void propagateFPControl(kmp_team_t *team) {
1024 if (__kmp_inherit_fp_control) {
1025 kmp_int16 x87_fpu_control_word;
1029 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1030 __kmp_store_mxcsr(&mxcsr);
1031 mxcsr &= KMP_X86_MXCSR_MASK;
1042 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1043 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1046 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1050 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1056 inline static void updateHWFPControl(kmp_team_t *team) {
1057 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1060 kmp_int16 x87_fpu_control_word;
1062 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1063 __kmp_store_mxcsr(&mxcsr);
1064 mxcsr &= KMP_X86_MXCSR_MASK;
1066 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1067 __kmp_clear_x87_fpu_status_word();
1068 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1071 if (team->t.t_mxcsr != mxcsr) {
1072 __kmp_load_mxcsr(&team->t.t_mxcsr);
1077 #define propagateFPControl(x) ((void)0) 1078 #define updateHWFPControl(x) ((void)0) 1081 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
1086 void __kmp_serialized_parallel(
ident_t *loc, kmp_int32 global_tid) {
1087 kmp_info_t *this_thr;
1088 kmp_team_t *serial_team;
1090 KC_TRACE(10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid));
1097 if (!TCR_4(__kmp_init_parallel))
1098 __kmp_parallel_initialize();
1099 __kmp_resume_if_soft_paused();
1101 this_thr = __kmp_threads[global_tid];
1102 serial_team = this_thr->th.th_serial_team;
1105 KMP_DEBUG_ASSERT(serial_team);
1108 if (__kmp_tasking_mode != tskm_immediate_exec) {
1110 this_thr->th.th_task_team ==
1111 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1112 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1114 KA_TRACE(20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / " 1115 "team %p, new task_team = NULL\n",
1116 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1117 this_thr->th.th_task_team = NULL;
1120 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1121 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1122 proc_bind = proc_bind_false;
1123 }
else if (proc_bind == proc_bind_default) {
1126 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1129 this_thr->th.th_set_proc_bind = proc_bind_default;
1132 ompt_data_t ompt_parallel_data = ompt_data_none;
1133 ompt_data_t *implicit_task_data;
1134 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1135 if (ompt_enabled.enabled &&
1136 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1138 ompt_task_info_t *parent_task_info;
1139 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1141 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1142 if (ompt_enabled.ompt_callback_parallel_begin) {
1145 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1146 &(parent_task_info->task_data), &(parent_task_info->frame),
1147 &ompt_parallel_data, team_size,
1148 ompt_parallel_invoker_program | ompt_parallel_team, codeptr);
1151 #endif // OMPT_SUPPORT 1153 if (this_thr->th.th_team != serial_team) {
1155 int level = this_thr->th.th_team->t.t_level;
1157 if (serial_team->t.t_serialized) {
1160 kmp_team_t *new_team;
1162 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1165 __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1169 proc_bind, &this_thr->th.th_current_task->td_icvs,
1170 0 USE_NESTED_HOT_ARG(NULL));
1171 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1172 KMP_ASSERT(new_team);
1175 new_team->t.t_threads[0] = this_thr;
1176 new_team->t.t_parent = this_thr->th.th_team;
1177 serial_team = new_team;
1178 this_thr->th.th_serial_team = serial_team;
1182 (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1183 global_tid, serial_team));
1191 (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1192 global_tid, serial_team));
1196 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1197 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1198 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1199 serial_team->t.t_ident = loc;
1200 serial_team->t.t_serialized = 1;
1201 serial_team->t.t_nproc = 1;
1202 serial_team->t.t_parent = this_thr->th.th_team;
1203 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1204 this_thr->th.th_team = serial_team;
1205 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1207 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1208 this_thr->th.th_current_task));
1209 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1210 this_thr->th.th_current_task->td_flags.executing = 0;
1212 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1217 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1218 &this_thr->th.th_current_task->td_parent->td_icvs);
1222 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1223 this_thr->th.th_current_task->td_icvs.nproc =
1224 __kmp_nested_nth.nth[level + 1];
1227 if (__kmp_nested_proc_bind.used &&
1228 (level + 1 < __kmp_nested_proc_bind.used)) {
1229 this_thr->th.th_current_task->td_icvs.proc_bind =
1230 __kmp_nested_proc_bind.bind_types[level + 1];
1234 serial_team->t.t_pkfn = (microtask_t)(~0);
1236 this_thr->th.th_info.ds.ds_tid = 0;
1239 this_thr->th.th_team_nproc = 1;
1240 this_thr->th.th_team_master = this_thr;
1241 this_thr->th.th_team_serialized = 1;
1243 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1244 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1245 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator;
1247 propagateFPControl(serial_team);
1250 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1251 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1252 serial_team->t.t_dispatch->th_disp_buffer =
1253 (dispatch_private_info_t *)__kmp_allocate(
1254 sizeof(dispatch_private_info_t));
1256 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1263 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1264 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1265 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1266 ++serial_team->t.t_serialized;
1267 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1270 int level = this_thr->th.th_team->t.t_level;
1273 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1274 this_thr->th.th_current_task->td_icvs.nproc =
1275 __kmp_nested_nth.nth[level + 1];
1277 serial_team->t.t_level++;
1278 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level " 1279 "of serial team %p to %d\n",
1280 global_tid, serial_team, serial_team->t.t_level));
1283 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1285 dispatch_private_info_t *disp_buffer =
1286 (dispatch_private_info_t *)__kmp_allocate(
1287 sizeof(dispatch_private_info_t));
1288 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1289 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1291 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1295 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
1299 if (__kmp_display_affinity) {
1300 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1301 this_thr->th.th_prev_num_threads != 1) {
1303 __kmp_aux_display_affinity(global_tid, NULL);
1304 this_thr->th.th_prev_level = serial_team->t.t_level;
1305 this_thr->th.th_prev_num_threads = 1;
1309 if (__kmp_env_consistency_check)
1310 __kmp_push_parallel(global_tid, NULL);
1312 serial_team->t.ompt_team_info.master_return_address = codeptr;
1313 if (ompt_enabled.enabled &&
1314 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1315 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr =
1316 OMPT_GET_FRAME_ADDRESS(0);
1318 ompt_lw_taskteam_t lw_taskteam;
1319 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1320 &ompt_parallel_data, codeptr);
1322 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1326 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr);
1327 if (ompt_enabled.ompt_callback_implicit_task) {
1328 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1329 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
1330 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid),
1331 ompt_task_implicit);
1332 OMPT_CUR_TASK_INFO(this_thr)->thread_num =
1333 __kmp_tid_from_gtid(global_tid);
1337 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1338 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr =
1339 OMPT_GET_FRAME_ADDRESS(0);
1346 int __kmp_fork_call(
ident_t *loc,
int gtid,
1347 enum fork_context_e call_context,
1348 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1353 int master_this_cons;
1355 kmp_team_t *parent_team;
1356 kmp_info_t *master_th;
1360 int master_set_numthreads;
1364 #if KMP_NESTED_HOT_TEAMS 1365 kmp_hot_team_ptr_t **p_hot_teams;
1368 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
1371 KA_TRACE(20, (
"__kmp_fork_call: enter T#%d\n", gtid));
1372 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1375 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1377 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1378 __kmp_stkpadding += (short)((kmp_int64)dummy);
1384 if (!TCR_4(__kmp_init_parallel))
1385 __kmp_parallel_initialize();
1386 __kmp_resume_if_soft_paused();
1389 master_th = __kmp_threads[gtid];
1391 parent_team = master_th->th.th_team;
1392 master_tid = master_th->th.th_info.ds.ds_tid;
1393 master_this_cons = master_th->th.th_local.this_construct;
1394 root = master_th->th.th_root;
1395 master_active = root->r.r_active;
1396 master_set_numthreads = master_th->th.th_set_nproc;
1399 ompt_data_t ompt_parallel_data = ompt_data_none;
1400 ompt_data_t *parent_task_data;
1401 ompt_frame_t *ompt_frame;
1402 ompt_data_t *implicit_task_data;
1403 void *return_address = NULL;
1405 if (ompt_enabled.enabled) {
1406 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1408 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
1413 level = parent_team->t.t_level;
1415 active_level = parent_team->t.t_active_level;
1417 teams_level = master_th->th.th_teams_level;
1418 #if KMP_NESTED_HOT_TEAMS 1419 p_hot_teams = &master_th->th.th_hot_teams;
1420 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1421 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1422 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1423 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1425 (*p_hot_teams)[0].hot_team_nth = 1;
1430 if (ompt_enabled.enabled) {
1431 if (ompt_enabled.ompt_callback_parallel_begin) {
1432 int team_size = master_set_numthreads
1433 ? master_set_numthreads
1434 : get__nproc_2(parent_team, master_tid);
1435 int flags = OMPT_INVOKER(call_context) |
1436 ((microtask == (microtask_t)__kmp_teams_master)
1437 ? ompt_parallel_league
1438 : ompt_parallel_team);
1439 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1440 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags,
1443 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1447 master_th->th.th_ident = loc;
1449 if (master_th->th.th_teams_microtask && ap &&
1450 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1454 parent_team->t.t_ident = loc;
1455 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1456 parent_team->t.t_argc = argc;
1457 argv = (
void **)parent_team->t.t_argv;
1458 for (i = argc - 1; i >= 0; --i)
1459 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1461 if (parent_team == master_th->th.th_serial_team) {
1464 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1466 if (call_context == fork_context_gnu) {
1469 parent_team->t.t_serialized--;
1475 void **exit_frame_p;
1477 ompt_lw_taskteam_t lw_taskteam;
1479 if (ompt_enabled.enabled) {
1480 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1481 &ompt_parallel_data, return_address);
1482 exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
1484 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1488 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1489 if (ompt_enabled.ompt_callback_implicit_task) {
1490 OMPT_CUR_TASK_INFO(master_th)->thread_num =
1491 __kmp_tid_from_gtid(gtid);
1492 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1493 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1494 implicit_task_data, 1,
1495 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1499 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1501 exit_frame_p = &dummy;
1506 parent_team->t.t_serialized--;
1509 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1510 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1511 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1520 if (ompt_enabled.enabled) {
1521 *exit_frame_p = NULL;
1522 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;
1523 if (ompt_enabled.ompt_callback_implicit_task) {
1524 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1525 ompt_scope_end, NULL, implicit_task_data, 1,
1526 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1528 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1529 __ompt_lw_taskteam_unlink(master_th);
1530 if (ompt_enabled.ompt_callback_parallel_end) {
1531 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1532 &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th),
1533 OMPT_INVOKER(call_context) | ompt_parallel_team,
1536 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1542 parent_team->t.t_pkfn = microtask;
1543 parent_team->t.t_invoke = invoker;
1544 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1545 parent_team->t.t_active_level++;
1546 parent_team->t.t_level++;
1547 parent_team->t.t_def_allocator = master_th->th.th_def_allocator;
1550 if (ompt_enabled.enabled) {
1551 ompt_lw_taskteam_t lw_taskteam;
1552 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1553 &ompt_parallel_data, return_address);
1554 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1,
true);
1559 if (master_set_numthreads) {
1560 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1562 kmp_info_t **other_threads = parent_team->t.t_threads;
1563 parent_team->t.t_nproc = master_set_numthreads;
1564 for (i = 0; i < master_set_numthreads; ++i) {
1565 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1569 master_th->th.th_set_nproc = 0;
1573 if (__kmp_debugging) {
1574 int nth = __kmp_omp_num_threads(loc);
1576 master_set_numthreads = nth;
1581 #if USE_ITT_BUILD && USE_ITT_NOTIFY 1582 if (((__itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr) ||
1584 __kmp_forkjoin_frames_mode == 3 &&
1585 parent_team->t.t_active_level == 1
1586 && master_th->th.th_teams_size.nteams == 1) {
1587 kmp_uint64 tmp_time = __itt_get_timestamp();
1588 master_th->th.th_frame_time = tmp_time;
1589 parent_team->t.t_region_time = tmp_time;
1591 if (__itt_stack_caller_create_ptr) {
1592 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL);
1594 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
1598 KF_TRACE(10, (
"__kmp_fork_call: before internal fork: root=%p, team=%p, " 1599 "master_th=%p, gtid=%d\n",
1600 root, parent_team, master_th, gtid));
1601 __kmp_internal_fork(loc, gtid, parent_team);
1602 KF_TRACE(10, (
"__kmp_fork_call: after internal fork: root=%p, team=%p, " 1603 "master_th=%p, gtid=%d\n",
1604 root, parent_team, master_th, gtid));
1606 if (call_context == fork_context_gnu)
1610 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1611 parent_team->t.t_id, parent_team->t.t_pkfn));
1613 if (!parent_team->t.t_invoke(gtid)) {
1614 KMP_ASSERT2(0,
"cannot invoke microtask for PRIMARY thread");
1616 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1617 parent_team->t.t_id, parent_team->t.t_pkfn));
1620 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
1626 if (__kmp_tasking_mode != tskm_immediate_exec) {
1627 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1628 parent_team->t.t_task_team[master_th->th.th_task_state]);
1632 int enter_teams = 0;
1633 if (parent_team->t.t_active_level >=
1634 master_th->th.th_current_task->td_icvs.max_active_levels) {
1637 enter_teams = ((ap == NULL && active_level == 0) ||
1638 (ap && teams_level > 0 && teams_level == level));
1640 master_set_numthreads
1641 ? master_set_numthreads
1650 if ((get__max_active_levels(master_th) == 1 &&
1651 (root->r.r_in_parallel && !enter_teams)) ||
1652 (__kmp_library == library_serial)) {
1653 KC_TRACE(10, (
"__kmp_fork_call: T#%d serializing team; requested %d" 1661 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1666 nthreads = __kmp_reserve_threads(root, parent_team, master_tid,
1667 nthreads, enter_teams);
1668 if (nthreads == 1) {
1672 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1676 KMP_DEBUG_ASSERT(nthreads > 0);
1679 master_th->th.th_set_nproc = 0;
1682 if (nthreads == 1) {
1684 #if KMP_OS_LINUX && \ 1685 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 1688 void **args = (
void **)KMP_ALLOCA(argc *
sizeof(
void *));
1693 (
"__kmp_fork_call: T#%d serializing parallel region\n", gtid));
1697 if (call_context == fork_context_intel) {
1699 master_th->th.th_serial_team->t.t_ident = loc;
1702 master_th->th.th_serial_team->t.t_level--;
1707 void **exit_frame_p;
1708 ompt_task_info_t *task_info;
1710 ompt_lw_taskteam_t lw_taskteam;
1712 if (ompt_enabled.enabled) {
1713 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1714 &ompt_parallel_data, return_address);
1716 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1719 task_info = OMPT_CUR_TASK_INFO(master_th);
1720 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1721 if (ompt_enabled.ompt_callback_implicit_task) {
1722 OMPT_CUR_TASK_INFO(master_th)->thread_num =
1723 __kmp_tid_from_gtid(gtid);
1724 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1725 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1726 &(task_info->task_data), 1,
1727 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1728 ompt_task_implicit);
1732 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1734 exit_frame_p = &dummy;
1739 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1740 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1741 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1742 parent_team->t.t_argv
1751 if (ompt_enabled.enabled) {
1752 *exit_frame_p = NULL;
1753 if (ompt_enabled.ompt_callback_implicit_task) {
1754 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1755 ompt_scope_end, NULL, &(task_info->task_data), 1,
1756 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1757 ompt_task_implicit);
1759 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1760 __ompt_lw_taskteam_unlink(master_th);
1761 if (ompt_enabled.ompt_callback_parallel_end) {
1762 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1763 &ompt_parallel_data, parent_task_data,
1764 OMPT_INVOKER(call_context) | ompt_parallel_team,
1767 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1770 }
else if (microtask == (microtask_t)__kmp_teams_master) {
1771 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1772 master_th->th.th_serial_team);
1773 team = master_th->th.th_team;
1775 team->t.t_invoke = invoker;
1776 __kmp_alloc_argv_entries(argc, team, TRUE);
1777 team->t.t_argc = argc;
1778 argv = (
void **)team->t.t_argv;
1780 for (i = argc - 1; i >= 0; --i)
1781 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1783 for (i = 0; i < argc; ++i)
1785 argv[i] = parent_team->t.t_argv[i];
1793 if (ompt_enabled.enabled) {
1794 ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th);
1795 if (ompt_enabled.ompt_callback_implicit_task) {
1796 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1797 ompt_scope_end, NULL, &(task_info->task_data), 0,
1798 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_initial);
1800 if (ompt_enabled.ompt_callback_parallel_end) {
1801 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1802 &ompt_parallel_data, parent_task_data,
1803 OMPT_INVOKER(call_context) | ompt_parallel_league,
1806 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1811 for (i = argc - 1; i >= 0; --i)
1812 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1817 void **exit_frame_p;
1818 ompt_task_info_t *task_info;
1820 ompt_lw_taskteam_t lw_taskteam;
1822 if (ompt_enabled.enabled) {
1823 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1824 &ompt_parallel_data, return_address);
1825 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1827 task_info = OMPT_CUR_TASK_INFO(master_th);
1828 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1831 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1832 if (ompt_enabled.ompt_callback_implicit_task) {
1833 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1834 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1835 implicit_task_data, 1, __kmp_tid_from_gtid(gtid),
1836 ompt_task_implicit);
1837 OMPT_CUR_TASK_INFO(master_th)->thread_num =
1838 __kmp_tid_from_gtid(gtid);
1842 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1844 exit_frame_p = &dummy;
1849 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1850 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1851 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1860 if (ompt_enabled.enabled) {
1861 *exit_frame_p = NULL;
1862 if (ompt_enabled.ompt_callback_implicit_task) {
1863 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1864 ompt_scope_end, NULL, &(task_info->task_data), 1,
1865 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1866 ompt_task_implicit);
1869 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1870 __ompt_lw_taskteam_unlink(master_th);
1871 if (ompt_enabled.ompt_callback_parallel_end) {
1872 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1873 &ompt_parallel_data, parent_task_data,
1874 OMPT_INVOKER(call_context) | ompt_parallel_team,
1877 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1881 }
else if (call_context == fork_context_gnu) {
1883 ompt_lw_taskteam_t lwt;
1884 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1887 lwt.ompt_task_info.frame.exit_frame = ompt_data_none;
1888 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1893 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1896 KMP_ASSERT2(call_context < fork_context_last,
1897 "__kmp_fork_call: unknown fork_context parameter");
1900 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1907 KF_TRACE(10, (
"__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, " 1908 "curtask=%p, curtask_max_aclevel=%d\n",
1909 parent_team->t.t_active_level, master_th,
1910 master_th->th.th_current_task,
1911 master_th->th.th_current_task->td_icvs.max_active_levels));
1915 master_th->th.th_current_task->td_flags.executing = 0;
1917 if (!master_th->th.th_teams_microtask || level > teams_level) {
1919 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1923 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1924 if ((level + 1 < __kmp_nested_nth.used) &&
1925 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1926 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1932 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
1933 kmp_proc_bind_t proc_bind_icv =
1935 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1936 proc_bind = proc_bind_false;
1938 if (proc_bind == proc_bind_default) {
1941 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1947 if ((level + 1 < __kmp_nested_proc_bind.used) &&
1948 (__kmp_nested_proc_bind.bind_types[level + 1] !=
1949 master_th->th.th_current_task->td_icvs.proc_bind)) {
1950 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
1955 master_th->th.th_set_proc_bind = proc_bind_default;
1957 if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) {
1958 kmp_internal_control_t new_icvs;
1959 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
1960 new_icvs.next = NULL;
1961 if (nthreads_icv > 0) {
1962 new_icvs.nproc = nthreads_icv;
1964 if (proc_bind_icv != proc_bind_default) {
1965 new_icvs.proc_bind = proc_bind_icv;
1969 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
1970 team = __kmp_allocate_team(root, nthreads, nthreads,
1974 proc_bind, &new_icvs,
1975 argc USE_NESTED_HOT_ARG(master_th));
1978 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
1979 team = __kmp_allocate_team(root, nthreads, nthreads,
1984 &master_th->th.th_current_task->td_icvs,
1985 argc USE_NESTED_HOT_ARG(master_th));
1988 10, (
"__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
1991 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
1992 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
1993 KMP_CHECK_UPDATE(team->t.t_ident, loc);
1994 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
1995 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
1997 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2000 KMP_CHECK_UPDATE(team->t.t_invoke, invoker);
2002 if (!master_th->th.th_teams_microtask || level > teams_level) {
2003 int new_level = parent_team->t.t_level + 1;
2004 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2005 new_level = parent_team->t.t_active_level + 1;
2006 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2009 int new_level = parent_team->t.t_level;
2010 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2011 new_level = parent_team->t.t_active_level;
2012 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2014 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
2016 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
2018 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
2019 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2022 propagateFPControl(team);
2024 if (__kmp_tasking_mode != tskm_immediate_exec) {
2027 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2028 parent_team->t.t_task_team[master_th->th.th_task_state]);
2029 KA_TRACE(20, (
"__kmp_fork_call: Primary T#%d pushing task_team %p / team " 2030 "%p, new task_team %p / team %p\n",
2031 __kmp_gtid_from_thread(master_th),
2032 master_th->th.th_task_team, parent_team,
2033 team->t.t_task_team[master_th->th.th_task_state], team));
2035 if (active_level || master_th->th.th_task_team) {
2037 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2038 if (master_th->th.th_task_state_top >=
2039 master_th->th.th_task_state_stack_sz) {
2040 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2041 kmp_uint8 *old_stack, *new_stack;
2043 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2044 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2045 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2047 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2051 old_stack = master_th->th.th_task_state_memo_stack;
2052 master_th->th.th_task_state_memo_stack = new_stack;
2053 master_th->th.th_task_state_stack_sz = new_size;
2054 __kmp_free(old_stack);
2058 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2059 master_th->th.th_task_state;
2060 master_th->th.th_task_state_top++;
2061 #if KMP_NESTED_HOT_TEAMS 2062 if (master_th->th.th_hot_teams &&
2063 active_level < __kmp_hot_teams_max_level &&
2064 team == master_th->th.th_hot_teams[active_level].hot_team) {
2066 master_th->th.th_task_state =
2068 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2071 master_th->th.th_task_state = 0;
2072 #if KMP_NESTED_HOT_TEAMS 2076 #if !KMP_NESTED_HOT_TEAMS 2077 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2078 (team == root->r.r_hot_team));
2084 (
"__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2085 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2087 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2088 (team->t.t_master_tid == 0 &&
2089 (team->t.t_parent == root->r.r_root_team ||
2090 team->t.t_parent->t.t_serialized)));
2094 argv = (
void **)team->t.t_argv;
2096 for (i = argc - 1; i >= 0; --i) {
2097 void *new_argv = va_arg(kmp_va_deref(ap),
void *);
2098 KMP_CHECK_UPDATE(*argv, new_argv);
2102 for (i = 0; i < argc; ++i) {
2104 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2109 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
2110 if (!root->r.r_active)
2111 root->r.r_active = TRUE;
2113 __kmp_fork_team_threads(root, team, master_th, gtid);
2114 __kmp_setup_icv_copy(team, nthreads,
2115 &master_th->th.th_current_task->td_icvs, loc);
2118 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2121 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2124 if (team->t.t_active_level == 1
2125 && !master_th->th.th_teams_microtask) {
2127 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2128 (__kmp_forkjoin_frames_mode == 3 ||
2129 __kmp_forkjoin_frames_mode == 1)) {
2130 kmp_uint64 tmp_time = 0;
2131 if (__itt_get_timestamp_ptr)
2132 tmp_time = __itt_get_timestamp();
2134 master_th->th.th_frame_time = tmp_time;
2135 if (__kmp_forkjoin_frames_mode == 3)
2136 team->t.t_region_time = tmp_time;
2140 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2141 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2143 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2149 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
2152 (
"__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2153 root, team, master_th, gtid));
2156 if (__itt_stack_caller_create_ptr) {
2159 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL);
2160 team->t.t_stack_id = __kmp_itt_stack_caller_create();
2161 }
else if (parent_team->t.t_serialized) {
2166 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL);
2167 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
2175 __kmp_internal_fork(loc, gtid, team);
2176 KF_TRACE(10, (
"__kmp_internal_fork : after : root=%p, team=%p, " 2177 "master_th=%p, gtid=%d\n",
2178 root, team, master_th, gtid));
2181 if (call_context == fork_context_gnu) {
2182 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2187 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2188 team->t.t_id, team->t.t_pkfn));
2191 #if KMP_STATS_ENABLED 2195 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
2199 if (!team->t.t_invoke(gtid)) {
2200 KMP_ASSERT2(0,
"cannot invoke microtask for PRIMARY thread");
2203 #if KMP_STATS_ENABLED 2206 KMP_SET_THREAD_STATE(previous_state);
2210 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2211 team->t.t_id, team->t.t_pkfn));
2214 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2217 if (ompt_enabled.enabled) {
2218 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2226 static inline void __kmp_join_restore_state(kmp_info_t *thread,
2229 thread->th.ompt_thread_info.state =
2230 ((team->t.t_serialized) ? ompt_state_work_serial
2231 : ompt_state_work_parallel);
2234 static inline void __kmp_join_ompt(
int gtid, kmp_info_t *thread,
2235 kmp_team_t *team, ompt_data_t *parallel_data,
2236 int flags,
void *codeptr) {
2237 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2238 if (ompt_enabled.ompt_callback_parallel_end) {
2239 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2240 parallel_data, &(task_info->task_data), flags, codeptr);
2243 task_info->frame.enter_frame = ompt_data_none;
2244 __kmp_join_restore_state(thread, team);
2248 void __kmp_join_call(
ident_t *loc,
int gtid
2251 enum fork_context_e fork_context
2255 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2257 kmp_team_t *parent_team;
2258 kmp_info_t *master_th;
2262 KA_TRACE(20, (
"__kmp_join_call: enter T#%d\n", gtid));
2265 master_th = __kmp_threads[gtid];
2266 root = master_th->th.th_root;
2267 team = master_th->th.th_team;
2268 parent_team = team->t.t_parent;
2270 master_th->th.th_ident = loc;
2273 void *team_microtask = (
void *)team->t.t_pkfn;
2277 if (ompt_enabled.enabled &&
2278 !(team->t.t_serialized && fork_context == fork_context_gnu)) {
2279 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2284 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2285 KA_TRACE(20, (
"__kmp_join_call: T#%d, old team = %p old task_team = %p, " 2286 "th_task_team = %p\n",
2287 __kmp_gtid_from_thread(master_th), team,
2288 team->t.t_task_team[master_th->th.th_task_state],
2289 master_th->th.th_task_team));
2290 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2291 team->t.t_task_team[master_th->th.th_task_state]);
2295 if (team->t.t_serialized) {
2296 if (master_th->th.th_teams_microtask) {
2298 int level = team->t.t_level;
2299 int tlevel = master_th->th.th_teams_level;
2300 if (level == tlevel) {
2304 }
else if (level == tlevel + 1) {
2308 team->t.t_serialized++;
2314 if (ompt_enabled.enabled) {
2315 __kmp_join_restore_state(master_th, parent_team);
2322 master_active = team->t.t_master_active;
2327 __kmp_internal_join(loc, gtid, team);
2329 if (__itt_stack_caller_create_ptr) {
2330 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL);
2332 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id);
2333 team->t.t_stack_id = NULL;
2337 master_th->th.th_task_state =
2340 if (__itt_stack_caller_create_ptr && parent_team->t.t_serialized) {
2341 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id != NULL);
2345 __kmp_itt_stack_caller_destroy((__itt_caller)parent_team->t.t_stack_id);
2346 parent_team->t.t_stack_id = NULL;
2354 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2355 void *codeptr = team->t.ompt_team_info.master_return_address;
2360 if (team->t.t_active_level == 1 &&
2361 (!master_th->th.th_teams_microtask ||
2362 master_th->th.th_teams_size.nteams == 1)) {
2363 master_th->th.th_ident = loc;
2366 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2367 __kmp_forkjoin_frames_mode == 3)
2368 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2369 master_th->th.th_frame_time, 0, loc,
2370 master_th->th.th_team_nproc, 1);
2371 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2372 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2373 __kmp_itt_region_joined(gtid);
2377 if (master_th->th.th_teams_microtask && !exit_teams &&
2378 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2379 team->t.t_level == master_th->th.th_teams_level + 1) {
2384 ompt_data_t ompt_parallel_data = ompt_data_none;
2385 if (ompt_enabled.enabled) {
2386 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2387 if (ompt_enabled.ompt_callback_implicit_task) {
2388 int ompt_team_size = team->t.t_nproc;
2389 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2390 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2391 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
2393 task_info->frame.exit_frame = ompt_data_none;
2394 task_info->task_data = ompt_data_none;
2395 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
2396 __ompt_lw_taskteam_unlink(master_th);
2401 team->t.t_active_level--;
2402 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2408 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2409 int old_num = master_th->th.th_team_nproc;
2410 int new_num = master_th->th.th_teams_size.nth;
2411 kmp_info_t **other_threads = team->t.t_threads;
2412 team->t.t_nproc = new_num;
2413 for (
int i = 0; i < old_num; ++i) {
2414 other_threads[i]->th.th_team_nproc = new_num;
2417 for (
int i = old_num; i < new_num; ++i) {
2419 KMP_DEBUG_ASSERT(other_threads[i]);
2420 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2421 for (
int b = 0; b < bs_last_barrier; ++b) {
2422 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2423 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2425 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2428 if (__kmp_tasking_mode != tskm_immediate_exec) {
2430 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2436 if (ompt_enabled.enabled) {
2437 __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data,
2438 OMPT_INVOKER(fork_context) | ompt_parallel_team, codeptr);
2446 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2447 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2449 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2454 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2456 if (!master_th->th.th_teams_microtask ||
2457 team->t.t_level > master_th->th.th_teams_level) {
2459 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2461 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2464 if (ompt_enabled.enabled) {
2465 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2466 if (ompt_enabled.ompt_callback_implicit_task) {
2467 int flags = (team_microtask == (
void *)__kmp_teams_master)
2469 : ompt_task_implicit;
2470 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc;
2471 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2472 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2473 OMPT_CUR_TASK_INFO(master_th)->thread_num, flags);
2475 task_info->frame.exit_frame = ompt_data_none;
2476 task_info->task_data = ompt_data_none;
2480 KF_TRACE(10, (
"__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2482 __kmp_pop_current_task_from_thread(master_th);
2484 #if KMP_AFFINITY_SUPPORTED 2486 master_th->th.th_first_place = team->t.t_first_place;
2487 master_th->th.th_last_place = team->t.t_last_place;
2488 #endif // KMP_AFFINITY_SUPPORTED 2489 master_th->th.th_def_allocator = team->t.t_def_allocator;
2491 updateHWFPControl(team);
2493 if (root->r.r_active != master_active)
2494 root->r.r_active = master_active;
2496 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2504 master_th->th.th_team = parent_team;
2505 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2506 master_th->th.th_team_master = parent_team->t.t_threads[0];
2507 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2510 if (parent_team->t.t_serialized &&
2511 parent_team != master_th->th.th_serial_team &&
2512 parent_team != root->r.r_root_team) {
2513 __kmp_free_team(root,
2514 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2515 master_th->th.th_serial_team = parent_team;
2518 if (__kmp_tasking_mode != tskm_immediate_exec) {
2519 if (master_th->th.th_task_state_top >
2521 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2523 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2524 master_th->th.th_task_state;
2525 --master_th->th.th_task_state_top;
2527 master_th->th.th_task_state =
2529 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2532 master_th->th.th_task_team =
2533 parent_team->t.t_task_team[master_th->th.th_task_state];
2535 (
"__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n",
2536 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2543 master_th->th.th_current_task->td_flags.executing = 1;
2545 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2549 OMPT_INVOKER(fork_context) |
2550 ((team_microtask == (
void *)__kmp_teams_master) ? ompt_parallel_league
2551 : ompt_parallel_team);
2552 if (ompt_enabled.enabled) {
2553 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags,
2559 KA_TRACE(20, (
"__kmp_join_call: exit T#%d\n", gtid));
2564 void __kmp_save_internal_controls(kmp_info_t *thread) {
2566 if (thread->th.th_team != thread->th.th_serial_team) {
2569 if (thread->th.th_team->t.t_serialized > 1) {
2572 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2575 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2576 thread->th.th_team->t.t_serialized) {
2581 kmp_internal_control_t *control =
2582 (kmp_internal_control_t *)__kmp_allocate(
2583 sizeof(kmp_internal_control_t));
2585 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2587 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2589 control->next = thread->th.th_team->t.t_control_stack_top;
2590 thread->th.th_team->t.t_control_stack_top = control;
2596 void __kmp_set_num_threads(
int new_nth,
int gtid) {
2600 KF_TRACE(10, (
"__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2601 KMP_DEBUG_ASSERT(__kmp_init_serial);
2605 else if (new_nth > __kmp_max_nth)
2606 new_nth = __kmp_max_nth;
2609 thread = __kmp_threads[gtid];
2610 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2613 __kmp_save_internal_controls(thread);
2615 set__nproc(thread, new_nth);
2620 root = thread->th.th_root;
2621 if (__kmp_init_parallel && (!root->r.r_active) &&
2622 (root->r.r_hot_team->t.t_nproc > new_nth)
2623 #
if KMP_NESTED_HOT_TEAMS
2624 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2627 kmp_team_t *hot_team = root->r.r_hot_team;
2630 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2633 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2634 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2635 if (__kmp_tasking_mode != tskm_immediate_exec) {
2638 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2640 __kmp_free_thread(hot_team->t.t_threads[f]);
2641 hot_team->t.t_threads[f] = NULL;
2643 hot_team->t.t_nproc = new_nth;
2644 #if KMP_NESTED_HOT_TEAMS 2645 if (thread->th.th_hot_teams) {
2646 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2647 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2651 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2654 for (f = 0; f < new_nth; f++) {
2655 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2656 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2659 hot_team->t.t_size_changed = -1;
2664 void __kmp_set_max_active_levels(
int gtid,
int max_active_levels) {
2667 KF_TRACE(10, (
"__kmp_set_max_active_levels: new max_active_levels for thread " 2669 gtid, max_active_levels));
2670 KMP_DEBUG_ASSERT(__kmp_init_serial);
2673 if (max_active_levels < 0) {
2674 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2679 KF_TRACE(10, (
"__kmp_set_max_active_levels: the call is ignored: new " 2680 "max_active_levels for thread %d = (%d)\n",
2681 gtid, max_active_levels));
2684 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2689 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2690 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2691 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2697 KF_TRACE(10, (
"__kmp_set_max_active_levels: after validation: new " 2698 "max_active_levels for thread %d = (%d)\n",
2699 gtid, max_active_levels));
2701 thread = __kmp_threads[gtid];
2703 __kmp_save_internal_controls(thread);
2705 set__max_active_levels(thread, max_active_levels);
2709 int __kmp_get_max_active_levels(
int gtid) {
2712 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d\n", gtid));
2713 KMP_DEBUG_ASSERT(__kmp_init_serial);
2715 thread = __kmp_threads[gtid];
2716 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2717 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d, curtask=%p, " 2718 "curtask_maxaclevel=%d\n",
2719 gtid, thread->th.th_current_task,
2720 thread->th.th_current_task->td_icvs.max_active_levels));
2721 return thread->th.th_current_task->td_icvs.max_active_levels;
2725 void __kmp_set_num_teams(
int num_teams) {
2727 __kmp_nteams = num_teams;
2729 int __kmp_get_max_teams(
void) {
return __kmp_nteams; }
2731 void __kmp_set_teams_thread_limit(
int limit) {
2733 __kmp_teams_thread_limit = limit;
2735 int __kmp_get_teams_thread_limit(
void) {
return __kmp_teams_thread_limit; }
2737 KMP_BUILD_ASSERT(
sizeof(kmp_sched_t) ==
sizeof(
int));
2738 KMP_BUILD_ASSERT(
sizeof(
enum sched_type) ==
sizeof(
int));
2741 void __kmp_set_schedule(
int gtid, kmp_sched_t kind,
int chunk) {
2743 kmp_sched_t orig_kind;
2746 KF_TRACE(10, (
"__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2747 gtid, (
int)kind, chunk));
2748 KMP_DEBUG_ASSERT(__kmp_init_serial);
2755 kind = __kmp_sched_without_mods(kind);
2757 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2758 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2760 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2761 KMP_HNT(DefaultScheduleKindUsed,
"static, no chunk"),
2763 kind = kmp_sched_default;
2767 thread = __kmp_threads[gtid];
2769 __kmp_save_internal_controls(thread);
2771 if (kind < kmp_sched_upper_std) {
2772 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2775 thread->th.th_current_task->td_icvs.sched.r_sched_type =
kmp_sch_static;
2777 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2778 __kmp_sch_map[kind - kmp_sched_lower - 1];
2783 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2784 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2785 kmp_sched_lower - 2];
2787 __kmp_sched_apply_mods_intkind(
2788 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));
2789 if (kind == kmp_sched_auto || chunk < 1) {
2791 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2793 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2798 void __kmp_get_schedule(
int gtid, kmp_sched_t *kind,
int *chunk) {
2802 KF_TRACE(10, (
"__kmp_get_schedule: thread %d\n", gtid));
2803 KMP_DEBUG_ASSERT(__kmp_init_serial);
2805 thread = __kmp_threads[gtid];
2807 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2808 switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)) {
2810 case kmp_sch_static_greedy:
2811 case kmp_sch_static_balanced:
2812 *kind = kmp_sched_static;
2813 __kmp_sched_apply_mods_stdkind(kind, th_type);
2816 case kmp_sch_static_chunked:
2817 *kind = kmp_sched_static;
2819 case kmp_sch_dynamic_chunked:
2820 *kind = kmp_sched_dynamic;
2823 case kmp_sch_guided_iterative_chunked:
2824 case kmp_sch_guided_analytical_chunked:
2825 *kind = kmp_sched_guided;
2828 *kind = kmp_sched_auto;
2830 case kmp_sch_trapezoidal:
2831 *kind = kmp_sched_trapezoidal;
2833 #if KMP_STATIC_STEAL_ENABLED 2834 case kmp_sch_static_steal:
2835 *kind = kmp_sched_static_steal;
2839 KMP_FATAL(UnknownSchedulingType, th_type);
2842 __kmp_sched_apply_mods_stdkind(kind, th_type);
2843 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2846 int __kmp_get_ancestor_thread_num(
int gtid,
int level) {
2852 KF_TRACE(10, (
"__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2853 KMP_DEBUG_ASSERT(__kmp_init_serial);
2860 thr = __kmp_threads[gtid];
2861 team = thr->th.th_team;
2862 ii = team->t.t_level;
2866 if (thr->th.th_teams_microtask) {
2868 int tlevel = thr->th.th_teams_level;
2871 KMP_DEBUG_ASSERT(ii >= tlevel);
2883 return __kmp_tid_from_gtid(gtid);
2885 dd = team->t.t_serialized;
2887 while (ii > level) {
2888 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2890 if ((team->t.t_serialized) && (!dd)) {
2891 team = team->t.t_parent;
2895 team = team->t.t_parent;
2896 dd = team->t.t_serialized;
2901 return (dd > 1) ? (0) : (team->t.t_master_tid);
2904 int __kmp_get_team_size(
int gtid,
int level) {
2910 KF_TRACE(10, (
"__kmp_get_team_size: thread %d %d\n", gtid, level));
2911 KMP_DEBUG_ASSERT(__kmp_init_serial);
2918 thr = __kmp_threads[gtid];
2919 team = thr->th.th_team;
2920 ii = team->t.t_level;
2924 if (thr->th.th_teams_microtask) {
2926 int tlevel = thr->th.th_teams_level;
2929 KMP_DEBUG_ASSERT(ii >= tlevel);
2940 while (ii > level) {
2941 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2943 if (team->t.t_serialized && (!dd)) {
2944 team = team->t.t_parent;
2948 team = team->t.t_parent;
2953 return team->t.t_nproc;
2956 kmp_r_sched_t __kmp_get_schedule_global() {
2961 kmp_r_sched_t r_sched;
2967 enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched);
2968 enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched);
2971 r_sched.r_sched_type = __kmp_static;
2974 r_sched.r_sched_type = __kmp_guided;
2976 r_sched.r_sched_type = __kmp_sched;
2978 SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers);
2980 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
2982 r_sched.chunk = KMP_DEFAULT_CHUNK;
2984 r_sched.chunk = __kmp_chunk;
2992 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
int realloc) {
2994 KMP_DEBUG_ASSERT(team);
2995 if (!realloc || argc > team->t.t_max_argc) {
2997 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: needed entries=%d, " 2998 "current entries=%d\n",
2999 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3001 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3002 __kmp_free((
void *)team->t.t_argv);
3004 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3006 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3007 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: inline allocate %d " 3009 team->t.t_id, team->t.t_max_argc));
3010 team->t.t_argv = &team->t.t_inline_argv[0];
3011 if (__kmp_storage_map) {
3012 __kmp_print_storage_map_gtid(
3013 -1, &team->t.t_inline_argv[0],
3014 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3015 (
sizeof(
void *) * KMP_INLINE_ARGV_ENTRIES),
"team_%d.t_inline_argv",
3020 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3021 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3023 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: dynamic allocate %d " 3025 team->t.t_id, team->t.t_max_argc));
3027 (
void **)__kmp_page_allocate(
sizeof(
void *) * team->t.t_max_argc);
3028 if (__kmp_storage_map) {
3029 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3030 &team->t.t_argv[team->t.t_max_argc],
3031 sizeof(
void *) * team->t.t_max_argc,
3032 "team_%d.t_argv", team->t.t_id);
3038 static void __kmp_allocate_team_arrays(kmp_team_t *team,
int max_nth) {
3040 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3042 (kmp_info_t **)__kmp_allocate(
sizeof(kmp_info_t *) * max_nth);
3043 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3044 sizeof(dispatch_shared_info_t) * num_disp_buff);
3045 team->t.t_dispatch =
3046 (kmp_disp_t *)__kmp_allocate(
sizeof(kmp_disp_t) * max_nth);
3047 team->t.t_implicit_task_taskdata =
3048 (kmp_taskdata_t *)__kmp_allocate(
sizeof(kmp_taskdata_t) * max_nth);
3049 team->t.t_max_nproc = max_nth;
3052 for (i = 0; i < num_disp_buff; ++i) {
3053 team->t.t_disp_buffer[i].buffer_index = i;
3054 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3058 static void __kmp_free_team_arrays(kmp_team_t *team) {
3061 for (i = 0; i < team->t.t_max_nproc; ++i) {
3062 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3063 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3064 team->t.t_dispatch[i].th_disp_buffer = NULL;
3067 #if KMP_USE_HIER_SCHED 3068 __kmp_dispatch_free_hierarchies(team);
3070 __kmp_free(team->t.t_threads);
3071 __kmp_free(team->t.t_disp_buffer);
3072 __kmp_free(team->t.t_dispatch);
3073 __kmp_free(team->t.t_implicit_task_taskdata);
3074 team->t.t_threads = NULL;
3075 team->t.t_disp_buffer = NULL;
3076 team->t.t_dispatch = NULL;
3077 team->t.t_implicit_task_taskdata = 0;
3080 static void __kmp_reallocate_team_arrays(kmp_team_t *team,
int max_nth) {
3081 kmp_info_t **oldThreads = team->t.t_threads;
3083 __kmp_free(team->t.t_disp_buffer);
3084 __kmp_free(team->t.t_dispatch);
3085 __kmp_free(team->t.t_implicit_task_taskdata);
3086 __kmp_allocate_team_arrays(team, max_nth);
3088 KMP_MEMCPY(team->t.t_threads, oldThreads,
3089 team->t.t_nproc *
sizeof(kmp_info_t *));
3091 __kmp_free(oldThreads);
3094 static kmp_internal_control_t __kmp_get_global_icvs(
void) {
3096 kmp_r_sched_t r_sched =
3097 __kmp_get_schedule_global();
3099 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
3101 kmp_internal_control_t g_icvs = {
3103 (kmp_int8)__kmp_global.g.g_dynamic,
3105 (kmp_int8)__kmp_env_blocktime,
3107 __kmp_dflt_blocktime,
3112 __kmp_dflt_team_nth,
3116 __kmp_dflt_max_active_levels,
3120 __kmp_nested_proc_bind.bind_types[0],
3121 __kmp_default_device,
3128 static kmp_internal_control_t __kmp_get_x_global_icvs(
const kmp_team_t *team) {
3130 kmp_internal_control_t gx_icvs;
3131 gx_icvs.serial_nesting_level =
3133 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3134 gx_icvs.next = NULL;
3139 static void __kmp_initialize_root(kmp_root_t *root) {
3141 kmp_team_t *root_team;
3142 kmp_team_t *hot_team;
3143 int hot_team_max_nth;
3144 kmp_r_sched_t r_sched =
3145 __kmp_get_schedule_global();
3146 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3147 KMP_DEBUG_ASSERT(root);
3148 KMP_ASSERT(!root->r.r_begin);
3151 __kmp_init_lock(&root->r.r_begin_lock);
3152 root->r.r_begin = FALSE;
3153 root->r.r_active = FALSE;
3154 root->r.r_in_parallel = 0;
3155 root->r.r_blocktime = __kmp_dflt_blocktime;
3159 KF_TRACE(10, (
"__kmp_initialize_root: before root_team\n"));
3162 __kmp_allocate_team(root,
3168 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3170 USE_NESTED_HOT_ARG(NULL)
3175 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
3178 KF_TRACE(10, (
"__kmp_initialize_root: after root_team = %p\n", root_team));
3180 root->r.r_root_team = root_team;
3181 root_team->t.t_control_stack_top = NULL;
3184 root_team->t.t_threads[0] = NULL;
3185 root_team->t.t_nproc = 1;
3186 root_team->t.t_serialized = 1;
3188 root_team->t.t_sched.sched = r_sched.sched;
3191 (
"__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3192 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
3196 KF_TRACE(10, (
"__kmp_initialize_root: before hot_team\n"));
3199 __kmp_allocate_team(root,
3201 __kmp_dflt_team_nth_ub * 2,
3205 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3207 USE_NESTED_HOT_ARG(NULL)
3209 KF_TRACE(10, (
"__kmp_initialize_root: after hot_team = %p\n", hot_team));
3211 root->r.r_hot_team = hot_team;
3212 root_team->t.t_control_stack_top = NULL;
3215 hot_team->t.t_parent = root_team;
3218 hot_team_max_nth = hot_team->t.t_max_nproc;
3219 for (f = 0; f < hot_team_max_nth; ++f) {
3220 hot_team->t.t_threads[f] = NULL;
3222 hot_team->t.t_nproc = 1;
3224 hot_team->t.t_sched.sched = r_sched.sched;
3225 hot_team->t.t_size_changed = 0;
3230 typedef struct kmp_team_list_item {
3231 kmp_team_p
const *entry;
3232 struct kmp_team_list_item *next;
3233 } kmp_team_list_item_t;
3234 typedef kmp_team_list_item_t *kmp_team_list_t;
3236 static void __kmp_print_structure_team_accum(
3237 kmp_team_list_t list,
3238 kmp_team_p
const *team
3248 KMP_DEBUG_ASSERT(list != NULL);
3253 __kmp_print_structure_team_accum(list, team->t.t_parent);
3254 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3258 while (l->next != NULL && l->entry != team) {
3261 if (l->next != NULL) {
3267 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3273 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3274 sizeof(kmp_team_list_item_t));
3281 static void __kmp_print_structure_team(
char const *title, kmp_team_p
const *team
3284 __kmp_printf(
"%s", title);
3286 __kmp_printf(
"%2x %p\n", team->t.t_id, team);
3288 __kmp_printf(
" - (nil)\n");
3292 static void __kmp_print_structure_thread(
char const *title,
3293 kmp_info_p
const *thread) {
3294 __kmp_printf(
"%s", title);
3295 if (thread != NULL) {
3296 __kmp_printf(
"%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3298 __kmp_printf(
" - (nil)\n");
3302 void __kmp_print_structure(
void) {
3304 kmp_team_list_t list;
3308 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
sizeof(kmp_team_list_item_t));
3312 __kmp_printf(
"\n------------------------------\nGlobal Thread " 3313 "Table\n------------------------------\n");
3316 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3317 __kmp_printf(
"%2d", gtid);
3318 if (__kmp_threads != NULL) {
3319 __kmp_printf(
" %p", __kmp_threads[gtid]);
3321 if (__kmp_root != NULL) {
3322 __kmp_printf(
" %p", __kmp_root[gtid]);
3329 __kmp_printf(
"\n------------------------------\nThreads\n--------------------" 3331 if (__kmp_threads != NULL) {
3333 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3334 kmp_info_t
const *thread = __kmp_threads[gtid];
3335 if (thread != NULL) {
3336 __kmp_printf(
"GTID %2d %p:\n", gtid, thread);
3337 __kmp_printf(
" Our Root: %p\n", thread->th.th_root);
3338 __kmp_print_structure_team(
" Our Team: ", thread->th.th_team);
3339 __kmp_print_structure_team(
" Serial Team: ",
3340 thread->th.th_serial_team);
3341 __kmp_printf(
" Threads: %2d\n", thread->th.th_team_nproc);
3342 __kmp_print_structure_thread(
" Primary: ",
3343 thread->th.th_team_master);
3344 __kmp_printf(
" Serialized?: %2d\n", thread->th.th_team_serialized);
3345 __kmp_printf(
" Set NProc: %2d\n", thread->th.th_set_nproc);
3346 __kmp_printf(
" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3347 __kmp_print_structure_thread(
" Next in pool: ",
3348 thread->th.th_next_pool);
3350 __kmp_print_structure_team_accum(list, thread->th.th_team);
3351 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3355 __kmp_printf(
"Threads array is not allocated.\n");
3359 __kmp_printf(
"\n------------------------------\nUbers\n----------------------" 3361 if (__kmp_root != NULL) {
3363 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3364 kmp_root_t
const *root = __kmp_root[gtid];
3366 __kmp_printf(
"GTID %2d %p:\n", gtid, root);
3367 __kmp_print_structure_team(
" Root Team: ", root->r.r_root_team);
3368 __kmp_print_structure_team(
" Hot Team: ", root->r.r_hot_team);
3369 __kmp_print_structure_thread(
" Uber Thread: ",
3370 root->r.r_uber_thread);
3371 __kmp_printf(
" Active?: %2d\n", root->r.r_active);
3372 __kmp_printf(
" In Parallel: %2d\n",
3373 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
3375 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3376 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3380 __kmp_printf(
"Ubers array is not allocated.\n");
3383 __kmp_printf(
"\n------------------------------\nTeams\n----------------------" 3385 while (list->next != NULL) {
3386 kmp_team_p
const *team = list->entry;
3388 __kmp_printf(
"Team %2x %p:\n", team->t.t_id, team);
3389 __kmp_print_structure_team(
" Parent Team: ", team->t.t_parent);
3390 __kmp_printf(
" Primary TID: %2d\n", team->t.t_master_tid);
3391 __kmp_printf(
" Max threads: %2d\n", team->t.t_max_nproc);
3392 __kmp_printf(
" Levels of serial: %2d\n", team->t.t_serialized);
3393 __kmp_printf(
" Number threads: %2d\n", team->t.t_nproc);
3394 for (i = 0; i < team->t.t_nproc; ++i) {
3395 __kmp_printf(
" Thread %2d: ", i);
3396 __kmp_print_structure_thread(
"", team->t.t_threads[i]);
3398 __kmp_print_structure_team(
" Next in pool: ", team->t.t_next_pool);
3404 __kmp_printf(
"\n------------------------------\nPools\n----------------------" 3406 __kmp_print_structure_thread(
"Thread pool: ",
3407 CCAST(kmp_info_t *, __kmp_thread_pool));
3408 __kmp_print_structure_team(
"Team pool: ",
3409 CCAST(kmp_team_t *, __kmp_team_pool));
3413 while (list != NULL) {
3414 kmp_team_list_item_t *item = list;
3416 KMP_INTERNAL_FREE(item);
3425 static const unsigned __kmp_primes[] = {
3426 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3427 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3428 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3429 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3430 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3431 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3432 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3433 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3434 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3435 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3436 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3440 unsigned short __kmp_get_random(kmp_info_t *thread) {
3441 unsigned x = thread->th.th_x;
3442 unsigned short r = (
unsigned short)(x >> 16);
3444 thread->th.th_x = x * thread->th.th_a + 1;
3446 KA_TRACE(30, (
"__kmp_get_random: THREAD: %d, RETURN: %u\n",
3447 thread->th.th_info.ds.ds_tid, r));
3453 void __kmp_init_random(kmp_info_t *thread) {
3454 unsigned seed = thread->th.th_info.ds.ds_tid;
3457 __kmp_primes[seed % (
sizeof(__kmp_primes) /
sizeof(__kmp_primes[0]))];
3458 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3460 (
"__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
3466 static int __kmp_reclaim_dead_roots(
void) {
3469 for (i = 0; i < __kmp_threads_capacity; ++i) {
3470 if (KMP_UBER_GTID(i) &&
3471 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3474 r += __kmp_unregister_root_other_thread(i);
3499 static int __kmp_expand_threads(
int nNeed) {
3501 int minimumRequiredCapacity;
3503 kmp_info_t **newThreads;
3504 kmp_root_t **newRoot;
3510 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB 3513 added = __kmp_reclaim_dead_roots();
3542 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
3545 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3549 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3551 newCapacity = __kmp_threads_capacity;
3553 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3554 : __kmp_sys_max_nth;
3555 }
while (newCapacity < minimumRequiredCapacity);
3556 newThreads = (kmp_info_t **)__kmp_allocate(
3557 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3559 (kmp_root_t **)((
char *)newThreads +
sizeof(kmp_info_t *) * newCapacity);
3560 KMP_MEMCPY(newThreads, __kmp_threads,
3561 __kmp_threads_capacity *
sizeof(kmp_info_t *));
3562 KMP_MEMCPY(newRoot, __kmp_root,
3563 __kmp_threads_capacity *
sizeof(kmp_root_t *));
3565 kmp_info_t **temp_threads = __kmp_threads;
3566 *(kmp_info_t * *
volatile *)&__kmp_threads = newThreads;
3567 *(kmp_root_t * *
volatile *)&__kmp_root = newRoot;
3568 __kmp_free(temp_threads);
3569 added += newCapacity - __kmp_threads_capacity;
3570 *(
volatile int *)&__kmp_threads_capacity = newCapacity;
3572 if (newCapacity > __kmp_tp_capacity) {
3573 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3574 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3575 __kmp_threadprivate_resize_cache(newCapacity);
3577 *(
volatile int *)&__kmp_tp_capacity = newCapacity;
3579 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3588 int __kmp_register_root(
int initial_thread) {
3589 kmp_info_t *root_thread;
3593 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3594 KA_TRACE(20, (
"__kmp_register_root: entered\n"));
3611 capacity = __kmp_threads_capacity;
3612 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3619 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
3620 capacity -= __kmp_hidden_helper_threads_num;
3624 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3625 if (__kmp_tp_cached) {
3626 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3627 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3628 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
3630 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3640 if (TCR_4(__kmp_init_hidden_helper_threads)) {
3643 for (gtid = 1; TCR_PTR(__kmp_threads[gtid]) != NULL &&
3644 gtid <= __kmp_hidden_helper_threads_num;
3647 KMP_ASSERT(gtid <= __kmp_hidden_helper_threads_num);
3648 KA_TRACE(1, (
"__kmp_register_root: found slot in threads array for " 3649 "hidden helper thread: T#%d\n",
3655 if (initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3658 for (gtid = __kmp_hidden_helper_threads_num + 1;
3659 TCR_PTR(__kmp_threads[gtid]) != NULL; gtid++)
3663 1, (
"__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3664 KMP_ASSERT(gtid < __kmp_threads_capacity);
3669 TCW_4(__kmp_nth, __kmp_nth + 1);
3673 if (__kmp_adjust_gtid_mode) {
3674 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3675 if (TCR_4(__kmp_gtid_mode) != 2) {
3676 TCW_4(__kmp_gtid_mode, 2);
3679 if (TCR_4(__kmp_gtid_mode) != 1) {
3680 TCW_4(__kmp_gtid_mode, 1);
3685 #ifdef KMP_ADJUST_BLOCKTIME 3688 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3689 if (__kmp_nth > __kmp_avail_proc) {
3690 __kmp_zero_bt = TRUE;
3696 if (!(root = __kmp_root[gtid])) {
3697 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(
sizeof(kmp_root_t));
3698 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3701 #if KMP_STATS_ENABLED 3703 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3704 __kmp_stats_thread_ptr->startLife();
3705 KMP_SET_THREAD_STATE(SERIAL_REGION);
3708 __kmp_initialize_root(root);
3711 if (root->r.r_uber_thread) {
3712 root_thread = root->r.r_uber_thread;
3714 root_thread = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
3715 if (__kmp_storage_map) {
3716 __kmp_print_thread_storage_map(root_thread, gtid);
3718 root_thread->th.th_info.ds.ds_gtid = gtid;
3720 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
3722 root_thread->th.th_root = root;
3723 if (__kmp_env_consistency_check) {
3724 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3727 __kmp_initialize_fast_memory(root_thread);
3731 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3732 __kmp_initialize_bget(root_thread);
3734 __kmp_init_random(root_thread);
3738 if (!root_thread->th.th_serial_team) {
3739 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3740 KF_TRACE(10, (
"__kmp_register_root: before serial_team\n"));
3741 root_thread->th.th_serial_team = __kmp_allocate_team(
3746 proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3748 KMP_ASSERT(root_thread->th.th_serial_team);
3749 KF_TRACE(10, (
"__kmp_register_root: after serial_team = %p\n",
3750 root_thread->th.th_serial_team));
3753 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
3755 root->r.r_root_team->t.t_threads[0] = root_thread;
3756 root->r.r_hot_team->t.t_threads[0] = root_thread;
3757 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3759 root_thread->th.th_serial_team->t.t_serialized = 0;
3760 root->r.r_uber_thread = root_thread;
3763 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3764 TCW_4(__kmp_init_gtid, TRUE);
3767 __kmp_gtid_set_specific(gtid);
3770 __kmp_itt_thread_name(gtid);
3773 #ifdef KMP_TDATA_GTID 3776 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3777 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3779 KA_TRACE(20, (
"__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, " 3781 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3782 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3783 KMP_INIT_BARRIER_STATE));
3786 for (b = 0; b < bs_last_barrier; ++b) {
3787 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3789 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3793 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3794 KMP_INIT_BARRIER_STATE);
3796 #if KMP_AFFINITY_SUPPORTED 3797 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3798 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3799 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3800 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3801 if (TCR_4(__kmp_init_middle)) {
3802 __kmp_affinity_set_init_mask(gtid, TRUE);
3805 root_thread->th.th_def_allocator = __kmp_def_allocator;
3806 root_thread->th.th_prev_level = 0;
3807 root_thread->th.th_prev_num_threads = 1;
3809 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
3810 tmp->cg_root = root_thread;
3811 tmp->cg_thread_limit = __kmp_cg_max_nth;
3812 tmp->cg_nthreads = 1;
3813 KA_TRACE(100, (
"__kmp_register_root: Thread %p created node %p with" 3814 " cg_nthreads init to 1\n",
3817 root_thread->th.th_cg_roots = tmp;
3819 __kmp_root_counter++;
3822 if (!initial_thread && ompt_enabled.enabled) {
3824 kmp_info_t *root_thread = ompt_get_thread();
3826 ompt_set_thread_state(root_thread, ompt_state_overhead);
3828 if (ompt_enabled.ompt_callback_thread_begin) {
3829 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3830 ompt_thread_initial, __ompt_get_thread_data_internal());
3832 ompt_data_t *task_data;
3833 ompt_data_t *parallel_data;
3834 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data,
3836 if (ompt_enabled.ompt_callback_implicit_task) {
3837 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
3838 ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial);
3841 ompt_set_thread_state(root_thread, ompt_state_work_serial);
3846 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3851 #if KMP_NESTED_HOT_TEAMS 3852 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr,
int level,
3853 const int max_level) {
3855 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3856 if (!hot_teams || !hot_teams[level].hot_team) {
3859 KMP_DEBUG_ASSERT(level < max_level);
3860 kmp_team_t *team = hot_teams[level].hot_team;
3861 nth = hot_teams[level].hot_team_nth;
3863 if (level < max_level - 1) {
3864 for (i = 0; i < nth; ++i) {
3865 kmp_info_t *th = team->t.t_threads[i];
3866 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3867 if (i > 0 && th->th.th_hot_teams) {
3868 __kmp_free(th->th.th_hot_teams);
3869 th->th.th_hot_teams = NULL;
3873 __kmp_free_team(root, team, NULL);
3880 static int __kmp_reset_root(
int gtid, kmp_root_t *root) {
3881 kmp_team_t *root_team = root->r.r_root_team;
3882 kmp_team_t *hot_team = root->r.r_hot_team;
3883 int n = hot_team->t.t_nproc;
3886 KMP_DEBUG_ASSERT(!root->r.r_active);
3888 root->r.r_root_team = NULL;
3889 root->r.r_hot_team = NULL;
3892 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
3893 #if KMP_NESTED_HOT_TEAMS 3894 if (__kmp_hot_teams_max_level >
3896 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3897 kmp_info_t *th = hot_team->t.t_threads[i];
3898 if (__kmp_hot_teams_max_level > 1) {
3899 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3901 if (th->th.th_hot_teams) {
3902 __kmp_free(th->th.th_hot_teams);
3903 th->th.th_hot_teams = NULL;
3908 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
3913 if (__kmp_tasking_mode != tskm_immediate_exec) {
3914 __kmp_wait_to_unref_task_teams();
3920 10, (
"__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3922 (LPVOID) & (root->r.r_uber_thread->th),
3923 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3924 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3928 ompt_data_t *task_data;
3929 ompt_data_t *parallel_data;
3930 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data,
3932 if (ompt_enabled.ompt_callback_implicit_task) {
3933 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
3934 ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial);
3936 if (ompt_enabled.ompt_callback_thread_end) {
3937 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3938 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
3944 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
3945 KA_TRACE(100, (
"__kmp_reset_root: Thread %p decrement cg_nthreads on node %p" 3947 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,
3948 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads));
3951 KMP_DEBUG_ASSERT(root->r.r_uber_thread ==
3952 root->r.r_uber_thread->th.th_cg_roots->cg_root);
3953 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL);
3954 __kmp_free(root->r.r_uber_thread->th.th_cg_roots);
3955 root->r.r_uber_thread->th.th_cg_roots = NULL;
3957 __kmp_reap_thread(root->r.r_uber_thread, 1);
3961 root->r.r_uber_thread = NULL;
3963 root->r.r_begin = FALSE;
3968 void __kmp_unregister_root_current_thread(
int gtid) {
3969 KA_TRACE(1, (
"__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
3973 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3974 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
3975 KC_TRACE(10, (
"__kmp_unregister_root_current_thread: already finished, " 3978 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3981 kmp_root_t *root = __kmp_root[gtid];
3983 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
3984 KMP_ASSERT(KMP_UBER_GTID(gtid));
3985 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
3986 KMP_ASSERT(root->r.r_active == FALSE);
3990 kmp_info_t *thread = __kmp_threads[gtid];
3991 kmp_team_t *team = thread->th.th_team;
3992 kmp_task_team_t *task_team = thread->th.th_task_team;
3995 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
3998 thread->th.ompt_thread_info.state = ompt_state_undefined;
4000 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
4003 __kmp_reset_root(gtid, root);
4007 (
"__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
4009 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4016 static int __kmp_unregister_root_other_thread(
int gtid) {
4017 kmp_root_t *root = __kmp_root[gtid];
4020 KA_TRACE(1, (
"__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4021 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4022 KMP_ASSERT(KMP_UBER_GTID(gtid));
4023 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4024 KMP_ASSERT(root->r.r_active == FALSE);
4026 r = __kmp_reset_root(gtid, root);
4028 (
"__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4034 void __kmp_task_info() {
4036 kmp_int32 gtid = __kmp_entry_gtid();
4037 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4038 kmp_info_t *this_thr = __kmp_threads[gtid];
4039 kmp_team_t *steam = this_thr->th.th_serial_team;
4040 kmp_team_t *team = this_thr->th.th_team;
4043 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p " 4045 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4046 team->t.t_implicit_task_taskdata[tid].td_parent);
4053 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4054 int tid,
int gtid) {
4058 KMP_DEBUG_ASSERT(this_thr != NULL);
4059 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4060 KMP_DEBUG_ASSERT(team);
4061 KMP_DEBUG_ASSERT(team->t.t_threads);
4062 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4063 kmp_info_t *master = team->t.t_threads[0];
4064 KMP_DEBUG_ASSERT(master);
4065 KMP_DEBUG_ASSERT(master->th.th_root);
4069 TCW_SYNC_PTR(this_thr->th.th_team, team);
4071 this_thr->th.th_info.ds.ds_tid = tid;
4072 this_thr->th.th_set_nproc = 0;
4073 if (__kmp_tasking_mode != tskm_immediate_exec)
4076 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4078 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
4079 this_thr->th.th_set_proc_bind = proc_bind_default;
4080 #if KMP_AFFINITY_SUPPORTED 4081 this_thr->th.th_new_place = this_thr->th.th_current_place;
4083 this_thr->th.th_root = master->th.th_root;
4086 this_thr->th.th_team_nproc = team->t.t_nproc;
4087 this_thr->th.th_team_master = master;
4088 this_thr->th.th_team_serialized = team->t.t_serialized;
4089 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
4091 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
4093 KF_TRACE(10, (
"__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4094 tid, gtid, this_thr, this_thr->th.th_current_task));
4096 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4099 KF_TRACE(10, (
"__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4100 tid, gtid, this_thr, this_thr->th.th_current_task));
4105 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4107 this_thr->th.th_local.this_construct = 0;
4109 if (!this_thr->th.th_pri_common) {
4110 this_thr->th.th_pri_common =
4111 (
struct common_table *)__kmp_allocate(
sizeof(
struct common_table));
4112 if (__kmp_storage_map) {
4113 __kmp_print_storage_map_gtid(
4114 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4115 sizeof(
struct common_table),
"th_%d.th_pri_common\n", gtid);
4117 this_thr->th.th_pri_head = NULL;
4120 if (this_thr != master &&
4121 this_thr->th.th_cg_roots != master->th.th_cg_roots) {
4123 KMP_DEBUG_ASSERT(master->th.th_cg_roots);
4124 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
4127 int i = tmp->cg_nthreads--;
4128 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p decrement cg_nthreads" 4129 " on node %p of thread %p to %d\n",
4130 this_thr, tmp, tmp->cg_root, tmp->cg_nthreads));
4135 this_thr->th.th_cg_roots = master->th.th_cg_roots;
4137 this_thr->th.th_cg_roots->cg_nthreads++;
4138 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p increment cg_nthreads on" 4139 " node %p of thread %p to %d\n",
4140 this_thr, this_thr->th.th_cg_roots,
4141 this_thr->th.th_cg_roots->cg_root,
4142 this_thr->th.th_cg_roots->cg_nthreads));
4143 this_thr->th.th_current_task->td_icvs.thread_limit =
4144 this_thr->th.th_cg_roots->cg_thread_limit;
4149 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4152 sizeof(dispatch_private_info_t) *
4153 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4154 KD_TRACE(10, (
"__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4155 team->t.t_max_nproc));
4156 KMP_ASSERT(dispatch);
4157 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4158 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
4160 dispatch->th_disp_index = 0;
4161 dispatch->th_doacross_buf_idx = 0;
4162 if (!dispatch->th_disp_buffer) {
4163 dispatch->th_disp_buffer =
4164 (dispatch_private_info_t *)__kmp_allocate(disp_size);
4166 if (__kmp_storage_map) {
4167 __kmp_print_storage_map_gtid(
4168 gtid, &dispatch->th_disp_buffer[0],
4169 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4171 : __kmp_dispatch_num_buffers],
4173 "th_%d.th_dispatch.th_disp_buffer " 4174 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4175 gtid, team->t.t_id, gtid);
4178 memset(&dispatch->th_disp_buffer[0],
'\0', disp_size);
4181 dispatch->th_dispatch_pr_current = 0;
4182 dispatch->th_dispatch_sh_current = 0;
4184 dispatch->th_deo_fcn = 0;
4185 dispatch->th_dxo_fcn = 0;
4188 this_thr->th.th_next_pool = NULL;
4190 if (!this_thr->th.th_task_state_memo_stack) {
4192 this_thr->th.th_task_state_memo_stack =
4193 (kmp_uint8 *)__kmp_allocate(4 *
sizeof(kmp_uint8));
4194 this_thr->th.th_task_state_top = 0;
4195 this_thr->th.th_task_state_stack_sz = 4;
4196 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4198 this_thr->th.th_task_state_memo_stack[i] = 0;
4201 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4202 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4212 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4214 kmp_team_t *serial_team;
4215 kmp_info_t *new_thr;
4218 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4219 KMP_DEBUG_ASSERT(root && team);
4220 #if !KMP_NESTED_HOT_TEAMS 4221 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
4226 if (__kmp_thread_pool) {
4227 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
4228 __kmp_thread_pool = (
volatile kmp_info_t *)new_thr->th.th_next_pool;
4229 if (new_thr == __kmp_thread_pool_insert_pt) {
4230 __kmp_thread_pool_insert_pt = NULL;
4232 TCW_4(new_thr->th.th_in_pool, FALSE);
4233 __kmp_suspend_initialize_thread(new_thr);
4234 __kmp_lock_suspend_mx(new_thr);
4235 if (new_thr->th.th_active_in_pool == TRUE) {
4236 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE);
4237 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
4238 new_thr->th.th_active_in_pool = FALSE;
4240 __kmp_unlock_suspend_mx(new_thr);
4242 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d using thread T#%d\n",
4243 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4244 KMP_ASSERT(!new_thr->th.th_team);
4245 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4248 __kmp_initialize_info(new_thr, team, new_tid,
4249 new_thr->th.th_info.ds.ds_gtid);
4250 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
4252 TCW_4(__kmp_nth, __kmp_nth + 1);
4254 new_thr->th.th_task_state = 0;
4255 new_thr->th.th_task_state_top = 0;
4256 new_thr->th.th_task_state_stack_sz = 4;
4258 #ifdef KMP_ADJUST_BLOCKTIME 4261 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4262 if (__kmp_nth > __kmp_avail_proc) {
4263 __kmp_zero_bt = TRUE;
4272 kmp_balign_t *balign = new_thr->th.th_bar;
4273 for (b = 0; b < bs_last_barrier; ++b)
4274 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
4277 KF_TRACE(10, (
"__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4278 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
4285 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4286 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4291 if (!TCR_4(__kmp_init_monitor)) {
4292 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4293 if (!TCR_4(__kmp_init_monitor)) {
4294 KF_TRACE(10, (
"before __kmp_create_monitor\n"));
4295 TCW_4(__kmp_init_monitor, 1);
4296 __kmp_create_monitor(&__kmp_monitor);
4297 KF_TRACE(10, (
"after __kmp_create_monitor\n"));
4308 while (TCR_4(__kmp_init_monitor) < 2) {
4311 KF_TRACE(10, (
"after monitor thread has started\n"));
4314 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4321 int new_start_gtid = TCR_4(__kmp_init_hidden_helper_threads)
4323 : __kmp_hidden_helper_threads_num + 1;
4325 for (new_gtid = new_start_gtid; TCR_PTR(__kmp_threads[new_gtid]) != NULL;
4327 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4330 if (TCR_4(__kmp_init_hidden_helper_threads)) {
4331 KMP_DEBUG_ASSERT(new_gtid <= __kmp_hidden_helper_threads_num);
4336 new_thr = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
4338 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4340 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG 4343 __itt_suppress_mark_range(
4344 __itt_suppress_range, __itt_suppress_threading_errors,
4345 &new_thr->th.th_sleep_loc,
sizeof(new_thr->th.th_sleep_loc));
4346 __itt_suppress_mark_range(
4347 __itt_suppress_range, __itt_suppress_threading_errors,
4348 &new_thr->th.th_reap_state,
sizeof(new_thr->th.th_reap_state));
4350 __itt_suppress_mark_range(
4351 __itt_suppress_range, __itt_suppress_threading_errors,
4352 &new_thr->th.th_suspend_init,
sizeof(new_thr->th.th_suspend_init));
4354 __itt_suppress_mark_range(__itt_suppress_range,
4355 __itt_suppress_threading_errors,
4356 &new_thr->th.th_suspend_init_count,
4357 sizeof(new_thr->th.th_suspend_init_count));
4360 __itt_suppress_mark_range(__itt_suppress_range,
4361 __itt_suppress_threading_errors,
4362 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go),
4363 sizeof(new_thr->th.th_bar[0].bb.b_go));
4364 __itt_suppress_mark_range(__itt_suppress_range,
4365 __itt_suppress_threading_errors,
4366 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go),
4367 sizeof(new_thr->th.th_bar[1].bb.b_go));
4368 __itt_suppress_mark_range(__itt_suppress_range,
4369 __itt_suppress_threading_errors,
4370 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go),
4371 sizeof(new_thr->th.th_bar[2].bb.b_go));
4373 if (__kmp_storage_map) {
4374 __kmp_print_thread_storage_map(new_thr, new_gtid);
4379 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4380 KF_TRACE(10, (
"__kmp_allocate_thread: before th_serial/serial_team\n"));
4381 new_thr->th.th_serial_team = serial_team =
4382 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4386 proc_bind_default, &r_icvs,
4387 0 USE_NESTED_HOT_ARG(NULL));
4389 KMP_ASSERT(serial_team);
4390 serial_team->t.t_serialized = 0;
4392 serial_team->t.t_threads[0] = new_thr;
4394 (
"__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4398 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4401 __kmp_initialize_fast_memory(new_thr);
4405 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4406 __kmp_initialize_bget(new_thr);
4409 __kmp_init_random(new_thr);
4413 (
"__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4414 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4417 kmp_balign_t *balign = new_thr->th.th_bar;
4418 for (b = 0; b < bs_last_barrier; ++b) {
4419 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4420 balign[b].bb.team = NULL;
4421 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4422 balign[b].bb.use_oncore_barrier = 0;
4425 new_thr->th.th_spin_here = FALSE;
4426 new_thr->th.th_next_waiting = 0;
4428 new_thr->th.th_blocking =
false;
4431 #if KMP_AFFINITY_SUPPORTED 4432 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4433 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4434 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4435 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4437 new_thr->th.th_def_allocator = __kmp_def_allocator;
4438 new_thr->th.th_prev_level = 0;
4439 new_thr->th.th_prev_num_threads = 1;
4441 TCW_4(new_thr->th.th_in_pool, FALSE);
4442 new_thr->th.th_active_in_pool = FALSE;
4443 TCW_4(new_thr->th.th_active, TRUE);
4451 if (__kmp_adjust_gtid_mode) {
4452 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4453 if (TCR_4(__kmp_gtid_mode) != 2) {
4454 TCW_4(__kmp_gtid_mode, 2);
4457 if (TCR_4(__kmp_gtid_mode) != 1) {
4458 TCW_4(__kmp_gtid_mode, 1);
4463 #ifdef KMP_ADJUST_BLOCKTIME 4466 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4467 if (__kmp_nth > __kmp_avail_proc) {
4468 __kmp_zero_bt = TRUE;
4475 10, (
"__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4476 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4478 (
"__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4480 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4491 static void __kmp_reinitialize_team(kmp_team_t *team,
4492 kmp_internal_control_t *new_icvs,
4494 KF_TRACE(10, (
"__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4495 team->t.t_threads[0], team));
4496 KMP_DEBUG_ASSERT(team && new_icvs);
4497 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4498 KMP_CHECK_UPDATE(team->t.t_ident, loc);
4500 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
4502 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4503 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4505 KF_TRACE(10, (
"__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4506 team->t.t_threads[0], team));
4512 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
4513 kmp_internal_control_t *new_icvs,
4515 KF_TRACE(10, (
"__kmp_initialize_team: enter: team=%p\n", team));
4518 KMP_DEBUG_ASSERT(team);
4519 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4520 KMP_DEBUG_ASSERT(team->t.t_threads);
4523 team->t.t_master_tid = 0;
4525 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4526 team->t.t_nproc = new_nproc;
4529 team->t.t_next_pool = NULL;
4533 TCW_SYNC_PTR(team->t.t_pkfn, NULL);
4534 team->t.t_invoke = NULL;
4537 team->t.t_sched.sched = new_icvs->sched.sched;
4539 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4540 team->t.t_fp_control_saved = FALSE;
4541 team->t.t_x87_fpu_control_word = 0;
4542 team->t.t_mxcsr = 0;
4545 team->t.t_construct = 0;
4547 team->t.t_ordered.dt.t_value = 0;
4548 team->t.t_master_active = FALSE;
4551 team->t.t_copypriv_data = NULL;
4554 team->t.t_copyin_counter = 0;
4557 team->t.t_control_stack_top = NULL;
4559 __kmp_reinitialize_team(team, new_icvs, loc);
4562 KF_TRACE(10, (
"__kmp_initialize_team: exit: team=%p\n", team));
4565 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 4568 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4569 if (KMP_AFFINITY_CAPABLE()) {
4571 if (old_mask != NULL) {
4572 status = __kmp_get_system_affinity(old_mask, TRUE);
4575 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4579 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4584 #if KMP_AFFINITY_SUPPORTED 4590 static void __kmp_partition_places(kmp_team_t *team,
int update_master_only) {
4592 kmp_info_t *master_th = team->t.t_threads[0];
4593 KMP_DEBUG_ASSERT(master_th != NULL);
4594 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4595 int first_place = master_th->th.th_first_place;
4596 int last_place = master_th->th.th_last_place;
4597 int masters_place = master_th->th.th_current_place;
4598 team->t.t_first_place = first_place;
4599 team->t.t_last_place = last_place;
4601 KA_TRACE(20, (
"__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) " 4602 "bound to place %d partition = [%d,%d]\n",
4603 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4604 team->t.t_id, masters_place, first_place, last_place));
4606 switch (proc_bind) {
4608 case proc_bind_default:
4611 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4614 case proc_bind_primary: {
4616 int n_th = team->t.t_nproc;
4617 for (f = 1; f < n_th; f++) {
4618 kmp_info_t *th = team->t.t_threads[f];
4619 KMP_DEBUG_ASSERT(th != NULL);
4620 th->th.th_first_place = first_place;
4621 th->th.th_last_place = last_place;
4622 th->th.th_new_place = masters_place;
4623 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4624 team->t.t_display_affinity != 1) {
4625 team->t.t_display_affinity = 1;
4628 KA_TRACE(100, (
"__kmp_partition_places: primary: T#%d(%d:%d) place %d " 4629 "partition = [%d,%d]\n",
4630 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4631 f, masters_place, first_place, last_place));
4635 case proc_bind_close: {
4637 int n_th = team->t.t_nproc;
4639 if (first_place <= last_place) {
4640 n_places = last_place - first_place + 1;
4642 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4644 if (n_th <= n_places) {
4645 int place = masters_place;
4646 for (f = 1; f < n_th; f++) {
4647 kmp_info_t *th = team->t.t_threads[f];
4648 KMP_DEBUG_ASSERT(th != NULL);
4650 if (place == last_place) {
4651 place = first_place;
4652 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4657 th->th.th_first_place = first_place;
4658 th->th.th_last_place = last_place;
4659 th->th.th_new_place = place;
4660 if (__kmp_display_affinity && place != th->th.th_current_place &&
4661 team->t.t_display_affinity != 1) {
4662 team->t.t_display_affinity = 1;
4665 KA_TRACE(100, (
"__kmp_partition_places: close: T#%d(%d:%d) place %d " 4666 "partition = [%d,%d]\n",
4667 __kmp_gtid_from_thread(team->t.t_threads[f]),
4668 team->t.t_id, f, place, first_place, last_place));
4671 int S, rem, gap, s_count;
4672 S = n_th / n_places;
4674 rem = n_th - (S * n_places);
4675 gap = rem > 0 ? n_places / rem : n_places;
4676 int place = masters_place;
4678 for (f = 0; f < n_th; f++) {
4679 kmp_info_t *th = team->t.t_threads[f];
4680 KMP_DEBUG_ASSERT(th != NULL);
4682 th->th.th_first_place = first_place;
4683 th->th.th_last_place = last_place;
4684 th->th.th_new_place = place;
4685 if (__kmp_display_affinity && place != th->th.th_current_place &&
4686 team->t.t_display_affinity != 1) {
4687 team->t.t_display_affinity = 1;
4691 if ((s_count == S) && rem && (gap_ct == gap)) {
4693 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4695 if (place == last_place) {
4696 place = first_place;
4697 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4705 }
else if (s_count == S) {
4706 if (place == last_place) {
4707 place = first_place;
4708 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4718 (
"__kmp_partition_places: close: T#%d(%d:%d) place %d " 4719 "partition = [%d,%d]\n",
4720 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4721 th->th.th_new_place, first_place, last_place));
4723 KMP_DEBUG_ASSERT(place == masters_place);
4727 case proc_bind_spread: {
4729 int n_th = team->t.t_nproc;
4732 if (first_place <= last_place) {
4733 n_places = last_place - first_place + 1;
4735 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4737 if (n_th <= n_places) {
4740 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4741 int S = n_places / n_th;
4742 int s_count, rem, gap, gap_ct;
4744 place = masters_place;
4745 rem = n_places - n_th * S;
4746 gap = rem ? n_th / rem : 1;
4749 if (update_master_only == 1)
4751 for (f = 0; f < thidx; f++) {
4752 kmp_info_t *th = team->t.t_threads[f];
4753 KMP_DEBUG_ASSERT(th != NULL);
4755 th->th.th_first_place = place;
4756 th->th.th_new_place = place;
4757 if (__kmp_display_affinity && place != th->th.th_current_place &&
4758 team->t.t_display_affinity != 1) {
4759 team->t.t_display_affinity = 1;
4762 while (s_count < S) {
4763 if (place == last_place) {
4764 place = first_place;
4765 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4772 if (rem && (gap_ct == gap)) {
4773 if (place == last_place) {
4774 place = first_place;
4775 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4783 th->th.th_last_place = place;
4786 if (place == last_place) {
4787 place = first_place;
4788 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4795 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4796 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4797 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4798 f, th->th.th_new_place, th->th.th_first_place,
4799 th->th.th_last_place, __kmp_affinity_num_masks));
4805 double current =
static_cast<double>(masters_place);
4807 (
static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4812 if (update_master_only == 1)
4814 for (f = 0; f < thidx; f++) {
4815 first =
static_cast<int>(current);
4816 last =
static_cast<int>(current + spacing) - 1;
4817 KMP_DEBUG_ASSERT(last >= first);
4818 if (first >= n_places) {
4819 if (masters_place) {
4822 if (first == (masters_place + 1)) {
4823 KMP_DEBUG_ASSERT(f == n_th);
4826 if (last == masters_place) {
4827 KMP_DEBUG_ASSERT(f == (n_th - 1));
4831 KMP_DEBUG_ASSERT(f == n_th);
4836 if (last >= n_places) {
4837 last = (n_places - 1);
4842 KMP_DEBUG_ASSERT(0 <= first);
4843 KMP_DEBUG_ASSERT(n_places > first);
4844 KMP_DEBUG_ASSERT(0 <= last);
4845 KMP_DEBUG_ASSERT(n_places > last);
4846 KMP_DEBUG_ASSERT(last_place >= first_place);
4847 th = team->t.t_threads[f];
4848 KMP_DEBUG_ASSERT(th);
4849 th->th.th_first_place = first;
4850 th->th.th_new_place = place;
4851 th->th.th_last_place = last;
4852 if (__kmp_display_affinity && place != th->th.th_current_place &&
4853 team->t.t_display_affinity != 1) {
4854 team->t.t_display_affinity = 1;
4857 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4858 "partition = [%d,%d], spacing = %.4f\n",
4859 __kmp_gtid_from_thread(team->t.t_threads[f]),
4860 team->t.t_id, f, th->th.th_new_place,
4861 th->th.th_first_place, th->th.th_last_place, spacing));
4865 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4867 int S, rem, gap, s_count;
4868 S = n_th / n_places;
4870 rem = n_th - (S * n_places);
4871 gap = rem > 0 ? n_places / rem : n_places;
4872 int place = masters_place;
4875 if (update_master_only == 1)
4877 for (f = 0; f < thidx; f++) {
4878 kmp_info_t *th = team->t.t_threads[f];
4879 KMP_DEBUG_ASSERT(th != NULL);
4881 th->th.th_first_place = place;
4882 th->th.th_last_place = place;
4883 th->th.th_new_place = place;
4884 if (__kmp_display_affinity && place != th->th.th_current_place &&
4885 team->t.t_display_affinity != 1) {
4886 team->t.t_display_affinity = 1;
4890 if ((s_count == S) && rem && (gap_ct == gap)) {
4892 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4894 if (place == last_place) {
4895 place = first_place;
4896 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4904 }
else if (s_count == S) {
4905 if (place == last_place) {
4906 place = first_place;
4907 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4916 KA_TRACE(100, (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d " 4917 "partition = [%d,%d]\n",
4918 __kmp_gtid_from_thread(team->t.t_threads[f]),
4919 team->t.t_id, f, th->th.th_new_place,
4920 th->th.th_first_place, th->th.th_last_place));
4922 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4930 KA_TRACE(20, (
"__kmp_partition_places: exit T#%d\n", team->t.t_id));
4933 #endif // KMP_AFFINITY_SUPPORTED 4938 __kmp_allocate_team(kmp_root_t *root,
int new_nproc,
int max_nproc,
4940 ompt_data_t ompt_parallel_data,
4942 kmp_proc_bind_t new_proc_bind,
4943 kmp_internal_control_t *new_icvs,
4944 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4945 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4948 int use_hot_team = !root->r.r_active;
4951 KA_TRACE(20, (
"__kmp_allocate_team: called\n"));
4952 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4953 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4956 #if KMP_NESTED_HOT_TEAMS 4957 kmp_hot_team_ptr_t *hot_teams;
4959 team = master->th.th_team;
4960 level = team->t.t_active_level;
4961 if (master->th.th_teams_microtask) {
4962 if (master->th.th_teams_size.nteams > 1 &&
4965 (microtask_t)__kmp_teams_master ||
4966 master->th.th_teams_level <
4972 hot_teams = master->th.th_hot_teams;
4973 if (level < __kmp_hot_teams_max_level && hot_teams &&
4974 hot_teams[level].hot_team) {
4982 KMP_DEBUG_ASSERT(new_nproc == 1);
4986 if (use_hot_team && new_nproc > 1) {
4987 KMP_DEBUG_ASSERT(new_nproc <= max_nproc);
4988 #if KMP_NESTED_HOT_TEAMS 4989 team = hot_teams[level].hot_team;
4991 team = root->r.r_hot_team;
4994 if (__kmp_tasking_mode != tskm_immediate_exec) {
4995 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p " 4996 "task_team[1] = %p before reinit\n",
4997 team->t.t_task_team[0], team->t.t_task_team[1]));
5004 if (team->t.t_nproc == new_nproc) {
5005 KA_TRACE(20, (
"__kmp_allocate_team: reusing hot team\n"));
5008 if (team->t.t_size_changed == -1) {
5009 team->t.t_size_changed = 1;
5011 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
5015 kmp_r_sched_t new_sched = new_icvs->sched;
5017 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
5019 __kmp_reinitialize_team(team, new_icvs,
5020 root->r.r_uber_thread->th.th_ident);
5022 KF_TRACE(10, (
"__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
5023 team->t.t_threads[0], team));
5024 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5026 #if KMP_AFFINITY_SUPPORTED 5027 if ((team->t.t_size_changed == 0) &&
5028 (team->t.t_proc_bind == new_proc_bind)) {
5029 if (new_proc_bind == proc_bind_spread) {
5030 __kmp_partition_places(
5033 KA_TRACE(200, (
"__kmp_allocate_team: reusing hot team #%d bindings: " 5034 "proc_bind = %d, partition = [%d,%d]\n",
5035 team->t.t_id, new_proc_bind, team->t.t_first_place,
5036 team->t.t_last_place));
5038 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5039 __kmp_partition_places(team);
5042 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5044 }
else if (team->t.t_nproc > new_nproc) {
5046 (
"__kmp_allocate_team: decreasing hot team thread count to %d\n",
5049 team->t.t_size_changed = 1;
5050 #if KMP_NESTED_HOT_TEAMS 5051 if (__kmp_hot_teams_mode == 0) {
5054 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
5055 hot_teams[level].hot_team_nth = new_nproc;
5056 #endif // KMP_NESTED_HOT_TEAMS 5058 for (f = new_nproc; f < team->t.t_nproc; f++) {
5059 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5060 if (__kmp_tasking_mode != tskm_immediate_exec) {
5063 team->t.t_threads[f]->th.th_task_team = NULL;
5065 __kmp_free_thread(team->t.t_threads[f]);
5066 team->t.t_threads[f] = NULL;
5068 #if KMP_NESTED_HOT_TEAMS 5073 for (f = new_nproc; f < team->t.t_nproc; ++f) {
5074 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5075 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
5076 for (
int b = 0; b < bs_last_barrier; ++b) {
5077 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
5078 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5080 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
5084 #endif // KMP_NESTED_HOT_TEAMS 5085 team->t.t_nproc = new_nproc;
5087 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
5088 __kmp_reinitialize_team(team, new_icvs,
5089 root->r.r_uber_thread->th.th_ident);
5092 for (f = 0; f < new_nproc; ++f) {
5093 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5098 KF_TRACE(10, (
"__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5099 team->t.t_threads[0], team));
5101 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5104 for (f = 0; f < team->t.t_nproc; f++) {
5105 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5106 team->t.t_threads[f]->th.th_team_nproc ==
5111 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5112 #if KMP_AFFINITY_SUPPORTED 5113 __kmp_partition_places(team);
5116 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5117 kmp_affin_mask_t *old_mask;
5118 if (KMP_AFFINITY_CAPABLE()) {
5119 KMP_CPU_ALLOC(old_mask);
5124 (
"__kmp_allocate_team: increasing hot team thread count to %d\n",
5127 team->t.t_size_changed = 1;
5129 #if KMP_NESTED_HOT_TEAMS 5130 int avail_threads = hot_teams[level].hot_team_nth;
5131 if (new_nproc < avail_threads)
5132 avail_threads = new_nproc;
5133 kmp_info_t **other_threads = team->t.t_threads;
5134 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5138 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5139 for (b = 0; b < bs_last_barrier; ++b) {
5140 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5141 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5143 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5147 if (hot_teams[level].hot_team_nth >= new_nproc) {
5150 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5151 team->t.t_nproc = new_nproc;
5157 hot_teams[level].hot_team_nth = new_nproc;
5158 #endif // KMP_NESTED_HOT_TEAMS 5159 if (team->t.t_max_nproc < new_nproc) {
5161 __kmp_reallocate_team_arrays(team, new_nproc);
5162 __kmp_reinitialize_team(team, new_icvs, NULL);
5165 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5171 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5175 for (f = team->t.t_nproc; f < new_nproc; f++) {
5176 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5177 KMP_DEBUG_ASSERT(new_worker);
5178 team->t.t_threads[f] = new_worker;
5181 (
"__kmp_allocate_team: team %d init T#%d arrived: " 5182 "join=%llu, plain=%llu\n",
5183 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5184 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5185 team->t.t_bar[bs_plain_barrier].b_arrived));
5189 kmp_balign_t *balign = new_worker->th.th_bar;
5190 for (b = 0; b < bs_last_barrier; ++b) {
5191 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5192 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5193 KMP_BARRIER_PARENT_FLAG);
5195 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5201 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED 5202 if (KMP_AFFINITY_CAPABLE()) {
5204 __kmp_set_system_affinity(old_mask, TRUE);
5205 KMP_CPU_FREE(old_mask);
5208 #if KMP_NESTED_HOT_TEAMS 5210 #endif // KMP_NESTED_HOT_TEAMS 5212 int old_nproc = team->t.t_nproc;
5214 __kmp_initialize_team(team, new_nproc, new_icvs,
5215 root->r.r_uber_thread->th.th_ident);
5218 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5219 for (f = 0; f < team->t.t_nproc; ++f)
5220 __kmp_initialize_info(team->t.t_threads[f], team, f,
5221 __kmp_gtid_from_tid(f, team));
5229 for (f = old_nproc; f < team->t.t_nproc; ++f)
5230 team->t.t_threads[f]->th.th_task_state =
5231 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5234 kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state;
5235 for (f = old_nproc; f < team->t.t_nproc; ++f)
5236 team->t.t_threads[f]->th.th_task_state = old_state;
5240 for (f = 0; f < team->t.t_nproc; ++f) {
5241 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5242 team->t.t_threads[f]->th.th_team_nproc ==
5247 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5248 #if KMP_AFFINITY_SUPPORTED 5249 __kmp_partition_places(team);
5253 kmp_info_t *master = team->t.t_threads[0];
5254 if (master->th.th_teams_microtask) {
5255 for (f = 1; f < new_nproc; ++f) {
5257 kmp_info_t *thr = team->t.t_threads[f];
5258 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5259 thr->th.th_teams_level = master->th.th_teams_level;
5260 thr->th.th_teams_size = master->th.th_teams_size;
5263 #if KMP_NESTED_HOT_TEAMS 5267 for (f = 1; f < new_nproc; ++f) {
5268 kmp_info_t *thr = team->t.t_threads[f];
5270 kmp_balign_t *balign = thr->th.th_bar;
5271 for (b = 0; b < bs_last_barrier; ++b) {
5272 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5273 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5275 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5280 #endif // KMP_NESTED_HOT_TEAMS 5283 __kmp_alloc_argv_entries(argc, team, TRUE);
5284 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5288 KF_TRACE(10, (
" hot_team = %p\n", team));
5291 if (__kmp_tasking_mode != tskm_immediate_exec) {
5292 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p " 5293 "task_team[1] = %p after reinit\n",
5294 team->t.t_task_team[0], team->t.t_task_team[1]));
5299 __ompt_team_assign_id(team, ompt_parallel_data);
5309 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
5312 if (team->t.t_max_nproc >= max_nproc) {
5314 __kmp_team_pool = team->t.t_next_pool;
5317 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5319 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and " 5320 "task_team[1] %p to NULL\n",
5321 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5322 team->t.t_task_team[0] = NULL;
5323 team->t.t_task_team[1] = NULL;
5326 __kmp_alloc_argv_entries(argc, team, TRUE);
5327 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5330 20, (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5331 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5334 for (b = 0; b < bs_last_barrier; ++b) {
5335 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5337 team->t.t_bar[b].b_master_arrived = 0;
5338 team->t.t_bar[b].b_team_arrived = 0;
5343 team->t.t_proc_bind = new_proc_bind;
5345 KA_TRACE(20, (
"__kmp_allocate_team: using team from pool %d.\n",
5349 __ompt_team_assign_id(team, ompt_parallel_data);
5361 team = __kmp_reap_team(team);
5362 __kmp_team_pool = team;
5367 team = (kmp_team_t *)__kmp_allocate(
sizeof(kmp_team_t));
5370 team->t.t_max_nproc = max_nproc;
5373 __kmp_allocate_team_arrays(team, max_nproc);
5375 KA_TRACE(20, (
"__kmp_allocate_team: making a new team\n"));
5376 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5378 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and task_team[1] " 5380 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5381 team->t.t_task_team[0] = NULL;
5383 team->t.t_task_team[1] = NULL;
5386 if (__kmp_storage_map) {
5387 __kmp_print_team_storage_map(
"team", team, team->t.t_id, new_nproc);
5391 __kmp_alloc_argv_entries(argc, team, FALSE);
5392 team->t.t_argc = argc;
5395 (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5396 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5399 for (b = 0; b < bs_last_barrier; ++b) {
5400 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5402 team->t.t_bar[b].b_master_arrived = 0;
5403 team->t.t_bar[b].b_team_arrived = 0;
5408 team->t.t_proc_bind = new_proc_bind;
5411 __ompt_team_assign_id(team, ompt_parallel_data);
5412 team->t.ompt_serialized_team_info = NULL;
5417 KA_TRACE(20, (
"__kmp_allocate_team: done creating a new team %d.\n",
5428 void __kmp_free_team(kmp_root_t *root,
5429 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5431 KA_TRACE(20, (
"__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5435 KMP_DEBUG_ASSERT(root);
5436 KMP_DEBUG_ASSERT(team);
5437 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5438 KMP_DEBUG_ASSERT(team->t.t_threads);
5440 int use_hot_team = team == root->r.r_hot_team;
5441 #if KMP_NESTED_HOT_TEAMS 5443 kmp_hot_team_ptr_t *hot_teams;
5445 level = team->t.t_active_level - 1;
5446 if (master->th.th_teams_microtask) {
5447 if (master->th.th_teams_size.nteams > 1) {
5451 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5452 master->th.th_teams_level == team->t.t_level) {
5457 hot_teams = master->th.th_hot_teams;
5458 if (level < __kmp_hot_teams_max_level) {
5459 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5463 #endif // KMP_NESTED_HOT_TEAMS 5466 TCW_SYNC_PTR(team->t.t_pkfn,
5469 team->t.t_copyin_counter = 0;
5474 if (!use_hot_team) {
5475 if (__kmp_tasking_mode != tskm_immediate_exec) {
5477 for (f = 1; f < team->t.t_nproc; ++f) {
5478 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5479 kmp_info_t *th = team->t.t_threads[f];
5480 volatile kmp_uint32 *state = &th->th.th_reap_state;
5481 while (*state != KMP_SAFE_TO_REAP) {
5485 if (!__kmp_is_thread_alive(th, &ecode)) {
5486 *state = KMP_SAFE_TO_REAP;
5491 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5492 if (fl.is_sleeping())
5493 fl.resume(__kmp_gtid_from_thread(th));
5500 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5501 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5502 if (task_team != NULL) {
5503 for (f = 0; f < team->t.t_nproc; ++f) {
5504 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5505 team->t.t_threads[f]->th.th_task_team = NULL;
5509 (
"__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5510 __kmp_get_gtid(), task_team, team->t.t_id));
5511 #if KMP_NESTED_HOT_TEAMS 5512 __kmp_free_task_team(master, task_team);
5514 team->t.t_task_team[tt_idx] = NULL;
5520 team->t.t_parent = NULL;
5521 team->t.t_level = 0;
5522 team->t.t_active_level = 0;
5525 for (f = 1; f < team->t.t_nproc; ++f) {
5526 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5527 __kmp_free_thread(team->t.t_threads[f]);
5528 team->t.t_threads[f] = NULL;
5533 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
5534 __kmp_team_pool = (
volatile kmp_team_t *)team;
5537 KMP_DEBUG_ASSERT(team->t.t_threads[1] &&
5538 team->t.t_threads[1]->th.th_cg_roots);
5539 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
5541 for (f = 1; f < team->t.t_nproc; ++f) {
5542 kmp_info_t *thr = team->t.t_threads[f];
5543 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&
5544 thr->th.th_cg_roots->cg_root == thr);
5546 kmp_cg_root_t *tmp = thr->th.th_cg_roots;
5547 thr->th.th_cg_roots = tmp->up;
5548 KA_TRACE(100, (
"__kmp_free_team: Thread %p popping node %p and moving" 5549 " up to node %p. cg_nthreads was %d\n",
5550 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads));
5551 int i = tmp->cg_nthreads--;
5556 if (thr->th.th_cg_roots)
5557 thr->th.th_current_task->td_icvs.thread_limit =
5558 thr->th.th_cg_roots->cg_thread_limit;
5567 kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5568 kmp_team_t *next_pool = team->t.t_next_pool;
5570 KMP_DEBUG_ASSERT(team);
5571 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5572 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5573 KMP_DEBUG_ASSERT(team->t.t_threads);
5574 KMP_DEBUG_ASSERT(team->t.t_argv);
5579 __kmp_free_team_arrays(team);
5580 if (team->t.t_argv != &team->t.t_inline_argv[0])
5581 __kmp_free((
void *)team->t.t_argv);
5613 void __kmp_free_thread(kmp_info_t *this_th) {
5617 KA_TRACE(20, (
"__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5618 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
5620 KMP_DEBUG_ASSERT(this_th);
5625 kmp_balign_t *balign = this_th->th.th_bar;
5626 for (b = 0; b < bs_last_barrier; ++b) {
5627 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5628 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5629 balign[b].bb.team = NULL;
5630 balign[b].bb.leaf_kids = 0;
5632 this_th->th.th_task_state = 0;
5633 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
5636 TCW_PTR(this_th->th.th_team, NULL);
5637 TCW_PTR(this_th->th.th_root, NULL);
5638 TCW_PTR(this_th->th.th_dispatch, NULL);
5640 while (this_th->th.th_cg_roots) {
5641 this_th->th.th_cg_roots->cg_nthreads--;
5642 KA_TRACE(100, (
"__kmp_free_thread: Thread %p decrement cg_nthreads on node" 5643 " %p of thread %p to %d\n",
5644 this_th, this_th->th.th_cg_roots,
5645 this_th->th.th_cg_roots->cg_root,
5646 this_th->th.th_cg_roots->cg_nthreads));
5647 kmp_cg_root_t *tmp = this_th->th.th_cg_roots;
5648 if (tmp->cg_root == this_th) {
5649 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0);
5651 5, (
"__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp));
5652 this_th->th.th_cg_roots = tmp->up;
5655 if (tmp->cg_nthreads == 0) {
5658 this_th->th.th_cg_roots = NULL;
5668 __kmp_free_implicit_task(this_th);
5669 this_th->th.th_current_task = NULL;
5673 gtid = this_th->th.th_info.ds.ds_gtid;
5674 if (__kmp_thread_pool_insert_pt != NULL) {
5675 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5676 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5677 __kmp_thread_pool_insert_pt = NULL;
5686 if (__kmp_thread_pool_insert_pt != NULL) {
5687 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5689 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
5691 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5692 scan = &((*scan)->th.th_next_pool))
5697 TCW_PTR(this_th->th.th_next_pool, *scan);
5698 __kmp_thread_pool_insert_pt = *scan = this_th;
5699 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5700 (this_th->th.th_info.ds.ds_gtid <
5701 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5702 TCW_4(this_th->th.th_in_pool, TRUE);
5703 __kmp_suspend_initialize_thread(this_th);
5704 __kmp_lock_suspend_mx(this_th);
5705 if (this_th->th.th_active == TRUE) {
5706 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
5707 this_th->th.th_active_in_pool = TRUE;
5711 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE);
5714 __kmp_unlock_suspend_mx(this_th);
5716 TCW_4(__kmp_nth, __kmp_nth - 1);
5718 #ifdef KMP_ADJUST_BLOCKTIME 5721 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5722 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5723 if (__kmp_nth <= __kmp_avail_proc) {
5724 __kmp_zero_bt = FALSE;
5734 void *__kmp_launch_thread(kmp_info_t *this_thr) {
5735 #if OMP_PROFILING_SUPPORT 5736 ProfileTraceFile = getenv(
"LIBOMPTARGET_PROFILE");
5738 if (ProfileTraceFile)
5739 llvm::timeTraceProfilerInitialize(500 ,
"libomptarget");
5742 int gtid = this_thr->th.th_info.ds.ds_gtid;
5744 kmp_team_t **
volatile pteam;
5747 KA_TRACE(10, (
"__kmp_launch_thread: T#%d start\n", gtid));
5749 if (__kmp_env_consistency_check) {
5750 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid);
5754 ompt_data_t *thread_data =
nullptr;
5755 if (ompt_enabled.enabled) {
5756 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
5757 *thread_data = ompt_data_none;
5759 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5760 this_thr->th.ompt_thread_info.wait_id = 0;
5761 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5762 this_thr->th.ompt_thread_info.parallel_flags = 0;
5763 if (ompt_enabled.ompt_callback_thread_begin) {
5764 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5765 ompt_thread_worker, thread_data);
5767 this_thr->th.ompt_thread_info.state = ompt_state_idle;
5772 while (!TCR_4(__kmp_global.g.g_done)) {
5773 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5777 KA_TRACE(20, (
"__kmp_launch_thread: T#%d waiting for work\n", gtid));
5780 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5783 if (ompt_enabled.enabled) {
5784 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5788 pteam = &this_thr->th.th_team;
5791 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
5793 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5796 (
"__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5797 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5798 (*pteam)->t.t_pkfn));
5800 updateHWFPControl(*pteam);
5803 if (ompt_enabled.enabled) {
5804 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
5808 rc = (*pteam)->t.t_invoke(gtid);
5812 KA_TRACE(20, (
"__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5813 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5814 (*pteam)->t.t_pkfn));
5817 if (ompt_enabled.enabled) {
5819 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;
5821 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5825 __kmp_join_barrier(gtid);
5828 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
5831 if (ompt_enabled.ompt_callback_thread_end) {
5832 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
5836 this_thr->th.th_task_team = NULL;
5838 __kmp_common_destroy_gtid(gtid);
5840 KA_TRACE(10, (
"__kmp_launch_thread: T#%d done\n", gtid));
5843 #if OMP_PROFILING_SUPPORT 5844 llvm::timeTraceProfilerFinishThread();
5851 void __kmp_internal_end_dest(
void *specific_gtid) {
5854 __kmp_type_convert((kmp_intptr_t)specific_gtid - 1, >id);
5856 KA_TRACE(30, (
"__kmp_internal_end_dest: T#%d\n", gtid));
5860 __kmp_internal_end_thread(gtid);
5863 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB 5865 __attribute__((destructor)) void __kmp_internal_end_dtor(
void) {
5866 __kmp_internal_end_atexit();
5873 void __kmp_internal_end_atexit(
void) {
5874 KA_TRACE(30, (
"__kmp_internal_end_atexit\n"));
5898 __kmp_internal_end_library(-1);
5900 __kmp_close_console();
5904 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root) {
5909 KMP_DEBUG_ASSERT(thread != NULL);
5911 gtid = thread->th.th_info.ds.ds_gtid;
5914 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5917 20, (
"__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5921 ANNOTATE_HAPPENS_BEFORE(thread);
5922 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
5924 __kmp_release_64(&flag);
5928 __kmp_reap_worker(thread);
5940 if (thread->th.th_active_in_pool) {
5941 thread->th.th_active_in_pool = FALSE;
5942 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
5943 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
5947 __kmp_free_implicit_task(thread);
5951 __kmp_free_fast_memory(thread);
5954 __kmp_suspend_uninitialize_thread(thread);
5956 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5957 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5962 #ifdef KMP_ADJUST_BLOCKTIME 5965 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5966 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5967 if (__kmp_nth <= __kmp_avail_proc) {
5968 __kmp_zero_bt = FALSE;
5974 if (__kmp_env_consistency_check) {
5975 if (thread->th.th_cons) {
5976 __kmp_free_cons_stack(thread->th.th_cons);
5977 thread->th.th_cons = NULL;
5981 if (thread->th.th_pri_common != NULL) {
5982 __kmp_free(thread->th.th_pri_common);
5983 thread->th.th_pri_common = NULL;
5986 if (thread->th.th_task_state_memo_stack != NULL) {
5987 __kmp_free(thread->th.th_task_state_memo_stack);
5988 thread->th.th_task_state_memo_stack = NULL;
5992 if (thread->th.th_local.bget_data != NULL) {
5993 __kmp_finalize_bget(thread);
5997 #if KMP_AFFINITY_SUPPORTED 5998 if (thread->th.th_affin_mask != NULL) {
5999 KMP_CPU_FREE(thread->th.th_affin_mask);
6000 thread->th.th_affin_mask = NULL;
6004 #if KMP_USE_HIER_SCHED 6005 if (thread->th.th_hier_bar_data != NULL) {
6006 __kmp_free(thread->th.th_hier_bar_data);
6007 thread->th.th_hier_bar_data = NULL;
6011 __kmp_reap_team(thread->th.th_serial_team);
6012 thread->th.th_serial_team = NULL;
6019 static void __kmp_internal_end(
void) {
6023 __kmp_unregister_library();
6030 __kmp_reclaim_dead_roots();
6034 for (i = 0; i < __kmp_threads_capacity; i++)
6036 if (__kmp_root[i]->r.r_active)
6039 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6041 if (i < __kmp_threads_capacity) {
6053 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6054 if (TCR_4(__kmp_init_monitor)) {
6055 __kmp_reap_monitor(&__kmp_monitor);
6056 TCW_4(__kmp_init_monitor, 0);
6058 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6059 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6060 #endif // KMP_USE_MONITOR 6065 for (i = 0; i < __kmp_threads_capacity; i++) {
6066 if (__kmp_root[i]) {
6069 KMP_ASSERT(!__kmp_root[i]->r.r_active);
6078 while (__kmp_thread_pool != NULL) {
6080 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
6081 __kmp_thread_pool = thread->th.th_next_pool;
6083 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
6084 thread->th.th_next_pool = NULL;
6085 thread->th.th_in_pool = FALSE;
6086 __kmp_reap_thread(thread, 0);
6088 __kmp_thread_pool_insert_pt = NULL;
6091 while (__kmp_team_pool != NULL) {
6093 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
6094 __kmp_team_pool = team->t.t_next_pool;
6096 team->t.t_next_pool = NULL;
6097 __kmp_reap_team(team);
6100 __kmp_reap_task_teams();
6107 for (i = 0; i < __kmp_threads_capacity; i++) {
6108 kmp_info_t *thr = __kmp_threads[i];
6109 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
6114 for (i = 0; i < __kmp_threads_capacity; ++i) {
6121 TCW_SYNC_4(__kmp_init_common, FALSE);
6123 KA_TRACE(10, (
"__kmp_internal_end: all workers reaped\n"));
6131 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6132 if (TCR_4(__kmp_init_monitor)) {
6133 __kmp_reap_monitor(&__kmp_monitor);
6134 TCW_4(__kmp_init_monitor, 0);
6136 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6137 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6140 TCW_4(__kmp_init_gtid, FALSE);
6149 void __kmp_internal_end_library(
int gtid_req) {
6156 if (__kmp_global.g.g_abort) {
6157 KA_TRACE(11, (
"__kmp_internal_end_library: abort, exiting\n"));
6161 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6162 KA_TRACE(10, (
"__kmp_internal_end_library: already finished\n"));
6169 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6171 10, (
"__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6172 if (gtid == KMP_GTID_SHUTDOWN) {
6173 KA_TRACE(10, (
"__kmp_internal_end_library: !__kmp_init_runtime, system " 6174 "already shutdown\n"));
6176 }
else if (gtid == KMP_GTID_MONITOR) {
6177 KA_TRACE(10, (
"__kmp_internal_end_library: monitor thread, gtid not " 6178 "registered, or system shutdown\n"));
6180 }
else if (gtid == KMP_GTID_DNE) {
6181 KA_TRACE(10, (
"__kmp_internal_end_library: gtid not registered or system " 6184 }
else if (KMP_UBER_GTID(gtid)) {
6186 if (__kmp_root[gtid]->r.r_active) {
6187 __kmp_global.g.g_abort = -1;
6188 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6189 __kmp_unregister_library();
6191 (
"__kmp_internal_end_library: root still active, abort T#%d\n",
6197 (
"__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6198 __kmp_unregister_root_current_thread(gtid);
6205 #ifdef DUMP_DEBUG_ON_EXIT 6206 if (__kmp_debug_buf)
6207 __kmp_dump_debug_buffer();
6212 __kmp_unregister_library();
6217 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6220 if (__kmp_global.g.g_abort) {
6221 KA_TRACE(10, (
"__kmp_internal_end_library: abort, exiting\n"));
6223 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6226 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6227 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6236 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6239 __kmp_internal_end();
6241 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6242 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6244 KA_TRACE(10, (
"__kmp_internal_end_library: exit\n"));
6246 #ifdef DUMP_DEBUG_ON_EXIT 6247 if (__kmp_debug_buf)
6248 __kmp_dump_debug_buffer();
6252 __kmp_close_console();
6255 __kmp_fini_allocator();
6259 void __kmp_internal_end_thread(
int gtid_req) {
6268 if (__kmp_global.g.g_abort) {
6269 KA_TRACE(11, (
"__kmp_internal_end_thread: abort, exiting\n"));
6273 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6274 KA_TRACE(10, (
"__kmp_internal_end_thread: already finished\n"));
6279 if (TCR_4(__kmp_init_hidden_helper)) {
6280 TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE);
6282 __kmp_hidden_helper_main_thread_release();
6284 __kmp_hidden_helper_threads_deinitz_wait();
6291 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6293 (
"__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6294 if (gtid == KMP_GTID_SHUTDOWN) {
6295 KA_TRACE(10, (
"__kmp_internal_end_thread: !__kmp_init_runtime, system " 6296 "already shutdown\n"));
6298 }
else if (gtid == KMP_GTID_MONITOR) {
6299 KA_TRACE(10, (
"__kmp_internal_end_thread: monitor thread, gtid not " 6300 "registered, or system shutdown\n"));
6302 }
else if (gtid == KMP_GTID_DNE) {
6303 KA_TRACE(10, (
"__kmp_internal_end_thread: gtid not registered or system " 6307 }
else if (KMP_UBER_GTID(gtid)) {
6309 if (__kmp_root[gtid]->r.r_active) {
6310 __kmp_global.g.g_abort = -1;
6311 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6313 (
"__kmp_internal_end_thread: root still active, abort T#%d\n",
6317 KA_TRACE(10, (
"__kmp_internal_end_thread: unregistering sibling T#%d\n",
6319 __kmp_unregister_root_current_thread(gtid);
6323 KA_TRACE(10, (
"__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6326 __kmp_threads[gtid]->th.th_task_team = NULL;
6330 (
"__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6336 if (__kmp_pause_status != kmp_hard_paused)
6340 KA_TRACE(10, (
"__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6345 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6348 if (__kmp_global.g.g_abort) {
6349 KA_TRACE(10, (
"__kmp_internal_end_thread: abort, exiting\n"));
6351 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6354 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6355 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6366 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6368 for (i = 0; i < __kmp_threads_capacity; ++i) {
6369 if (KMP_UBER_GTID(i)) {
6372 (
"__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6373 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6374 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6381 __kmp_internal_end();
6383 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6384 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6386 KA_TRACE(10, (
"__kmp_internal_end_thread: exit T#%d\n", gtid_req));
6388 #ifdef DUMP_DEBUG_ON_EXIT 6389 if (__kmp_debug_buf)
6390 __kmp_dump_debug_buffer();
6397 static long __kmp_registration_flag = 0;
6399 static char *__kmp_registration_str = NULL;
6402 static inline char *__kmp_reg_status_name() {
6408 #if KMP_OS_UNIX && !KMP_OS_DARWIN && KMP_DYNAMIC_LIB 6409 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d_%d", (
int)getpid(),
6412 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d", (
int)getpid());
6416 void __kmp_register_library_startup(
void) {
6418 char *name = __kmp_reg_status_name();
6424 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 6425 __kmp_initialize_system_tick();
6427 __kmp_read_system_time(&time.dtime);
6428 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6429 __kmp_registration_str =
6430 __kmp_str_format(
"%p-%lx-%s", &__kmp_registration_flag,
6431 __kmp_registration_flag, KMP_LIBRARY_FILE);
6433 KA_TRACE(50, (
"__kmp_register_library_startup: %s=\"%s\"\n", name,
6434 __kmp_registration_str));
6440 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6441 char *shm_name = __kmp_str_format(
"/%s", name);
6442 int shm_preexist = 0;
6444 int fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
6445 if ((fd1 == -1) && (errno == EEXIST)) {
6448 fd1 = shm_open(shm_name, O_RDWR, 0666);
6451 __kmp_fatal(KMP_MSG(FunctionError,
"Can't open SHM"), KMP_ERR(0),
6457 }
else if (fd1 == -1) {
6460 __kmp_fatal(KMP_MSG(FunctionError,
"Can't open SHM2"), KMP_ERR(errno),
6463 if (shm_preexist == 0) {
6465 if (ftruncate(fd1, SHM_SIZE) == -1) {
6467 __kmp_fatal(KMP_MSG(FunctionError,
"Can't set size of SHM"),
6468 KMP_ERR(errno), __kmp_msg_null);
6472 (
char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0);
6473 if (data1 == MAP_FAILED) {
6475 __kmp_fatal(KMP_MSG(FunctionError,
"Can't map SHM"), KMP_ERR(errno),
6478 if (shm_preexist == 0) {
6479 KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
6482 value = __kmp_str_format(
"%s", data1);
6483 munmap(data1, SHM_SIZE);
6485 #else // Windows and unix with static library 6487 __kmp_env_set(name, __kmp_registration_str, 0);
6489 value = __kmp_env_get(name);
6492 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6499 char *flag_addr_str = NULL;
6500 char *flag_val_str = NULL;
6501 char const *file_name = NULL;
6502 __kmp_str_split(tail,
'-', &flag_addr_str, &tail);
6503 __kmp_str_split(tail,
'-', &flag_val_str, &tail);
6506 long *flag_addr = 0;
6507 unsigned long flag_val = 0;
6508 KMP_SSCANF(flag_addr_str,
"%p", RCAST(
void **, &flag_addr));
6509 KMP_SSCANF(flag_val_str,
"%lx", &flag_val);
6510 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name,
"") != 0) {
6514 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6528 file_name =
"unknown library";
6533 char *duplicate_ok = __kmp_env_get(
"KMP_DUPLICATE_LIB_OK");
6534 if (!__kmp_str_match_true(duplicate_ok)) {
6536 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6537 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
6539 KMP_INTERNAL_FREE(duplicate_ok);
6540 __kmp_duplicate_library_ok = 1;
6545 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6547 shm_unlink(shm_name);
6550 __kmp_env_unset(name);
6554 KMP_DEBUG_ASSERT(0);
6558 KMP_INTERNAL_FREE((
void *)value);
6559 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6560 KMP_INTERNAL_FREE((
void *)shm_name);
6563 KMP_INTERNAL_FREE((
void *)name);
6567 void __kmp_unregister_library(
void) {
6569 char *name = __kmp_reg_status_name();
6572 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6573 char *shm_name = __kmp_str_format(
"/%s", name);
6574 int fd1 = shm_open(shm_name, O_RDONLY, 0666);
6579 char *data1 = (
char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
6580 if (data1 != MAP_FAILED) {
6581 value = __kmp_str_format(
"%s", data1);
6582 munmap(data1, SHM_SIZE);
6586 value = __kmp_env_get(name);
6589 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6590 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6591 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6593 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6594 shm_unlink(shm_name);
6596 __kmp_env_unset(name);
6600 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library 6601 KMP_INTERNAL_FREE(shm_name);
6604 KMP_INTERNAL_FREE(__kmp_registration_str);
6605 KMP_INTERNAL_FREE(value);
6606 KMP_INTERNAL_FREE(name);
6608 __kmp_registration_flag = 0;
6609 __kmp_registration_str = NULL;
6616 #if KMP_MIC_SUPPORTED 6618 static void __kmp_check_mic_type() {
6619 kmp_cpuid_t cpuid_state = {0};
6620 kmp_cpuid_t *cs_p = &cpuid_state;
6621 __kmp_x86_cpuid(1, 0, cs_p);
6623 if ((cs_p->eax & 0xff0) == 0xB10) {
6624 __kmp_mic_type = mic2;
6625 }
else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6626 __kmp_mic_type = mic3;
6628 __kmp_mic_type = non_mic;
6635 static void __kmp_user_level_mwait_init() {
6636 struct kmp_cpuid buf;
6637 __kmp_x86_cpuid(7, 0, &buf);
6638 __kmp_umwait_enabled = ((buf.ecx >> 5) & 1) && __kmp_user_level_mwait;
6639 KF_TRACE(30, (
"__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n",
6640 __kmp_umwait_enabled));
6642 #elif KMP_HAVE_MWAIT 6643 #ifndef AT_INTELPHIUSERMWAIT 6646 #define AT_INTELPHIUSERMWAIT 10000 6651 unsigned long getauxval(
unsigned long) KMP_WEAK_ATTRIBUTE_EXTERNAL;
6652 unsigned long getauxval(
unsigned long) {
return 0; }
6654 static void __kmp_user_level_mwait_init() {
6659 if (__kmp_mic_type == mic3) {
6660 unsigned long res = getauxval(AT_INTELPHIUSERMWAIT);
6661 if ((res & 0x1) || __kmp_user_level_mwait) {
6662 __kmp_mwait_enabled = TRUE;
6663 if (__kmp_user_level_mwait) {
6664 KMP_INFORM(EnvMwaitWarn);
6667 __kmp_mwait_enabled = FALSE;
6670 KF_TRACE(30, (
"__kmp_user_level_mwait_init: __kmp_mic_type = %d, " 6671 "__kmp_mwait_enabled = %d\n",
6672 __kmp_mic_type, __kmp_mwait_enabled));
6676 static void __kmp_do_serial_initialize(
void) {
6680 KA_TRACE(10, (
"__kmp_do_serial_initialize: enter\n"));
6682 KMP_DEBUG_ASSERT(
sizeof(kmp_int32) == 4);
6683 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) == 4);
6684 KMP_DEBUG_ASSERT(
sizeof(kmp_int64) == 8);
6685 KMP_DEBUG_ASSERT(
sizeof(kmp_uint64) == 8);
6686 KMP_DEBUG_ASSERT(
sizeof(kmp_intptr_t) ==
sizeof(
void *));
6692 __kmp_validate_locks();
6695 __kmp_init_allocator();
6700 __kmp_register_library_startup();
6703 if (TCR_4(__kmp_global.g.g_done)) {
6704 KA_TRACE(10, (
"__kmp_do_serial_initialize: reinitialization of library\n"));
6707 __kmp_global.g.g_abort = 0;
6708 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
6711 #if KMP_USE_ADAPTIVE_LOCKS 6712 #if KMP_DEBUG_ADAPTIVE_LOCKS 6713 __kmp_init_speculative_stats();
6716 #if KMP_STATS_ENABLED 6719 __kmp_init_lock(&__kmp_global_lock);
6720 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6721 __kmp_init_lock(&__kmp_debug_lock);
6722 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6723 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6724 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6725 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6726 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6727 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6728 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6729 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6730 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6731 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6732 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6733 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6734 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6735 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6736 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
6738 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
6740 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
6744 __kmp_runtime_initialize();
6746 #if KMP_MIC_SUPPORTED 6747 __kmp_check_mic_type();
6754 __kmp_abort_delay = 0;
6758 __kmp_dflt_team_nth_ub = __kmp_xproc;
6759 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6760 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6762 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6763 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6765 __kmp_max_nth = __kmp_sys_max_nth;
6766 __kmp_cg_max_nth = __kmp_sys_max_nth;
6767 __kmp_teams_max_nth = __kmp_xproc;
6768 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6769 __kmp_teams_max_nth = __kmp_sys_max_nth;
6774 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
6776 __kmp_monitor_wakeups =
6777 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6778 __kmp_bt_intervals =
6779 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6782 __kmp_library = library_throughput;
6784 __kmp_static = kmp_sch_static_balanced;
6791 #if KMP_FAST_REDUCTION_BARRIER 6792 #define kmp_reduction_barrier_gather_bb ((int)1) 6793 #define kmp_reduction_barrier_release_bb ((int)1) 6794 #define kmp_reduction_barrier_gather_pat bp_hyper_bar 6795 #define kmp_reduction_barrier_release_pat bp_hyper_bar 6796 #endif // KMP_FAST_REDUCTION_BARRIER 6797 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6798 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6799 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6800 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6801 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6802 #if KMP_FAST_REDUCTION_BARRIER 6803 if (i == bs_reduction_barrier) {
6805 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6806 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6807 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6808 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
6810 #endif // KMP_FAST_REDUCTION_BARRIER 6812 #if KMP_FAST_REDUCTION_BARRIER 6813 #undef kmp_reduction_barrier_release_pat 6814 #undef kmp_reduction_barrier_gather_pat 6815 #undef kmp_reduction_barrier_release_bb 6816 #undef kmp_reduction_barrier_gather_bb 6817 #endif // KMP_FAST_REDUCTION_BARRIER 6818 #if KMP_MIC_SUPPORTED 6819 if (__kmp_mic_type == mic2) {
6821 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3;
6822 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6824 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6825 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6827 #if KMP_FAST_REDUCTION_BARRIER 6828 if (__kmp_mic_type == mic2) {
6829 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6830 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6832 #endif // KMP_FAST_REDUCTION_BARRIER 6833 #endif // KMP_MIC_SUPPORTED 6837 __kmp_env_checks = TRUE;
6839 __kmp_env_checks = FALSE;
6843 __kmp_foreign_tp = TRUE;
6845 __kmp_global.g.g_dynamic = FALSE;
6846 __kmp_global.g.g_dynamic_mode = dynamic_default;
6848 __kmp_env_initialize(NULL);
6850 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT 6851 __kmp_user_level_mwait_init();
6855 char const *val = __kmp_env_get(
"KMP_DUMP_CATALOG");
6856 if (__kmp_str_match_true(val)) {
6857 kmp_str_buf_t buffer;
6858 __kmp_str_buf_init(&buffer);
6859 __kmp_i18n_dump_catalog(&buffer);
6860 __kmp_printf(
"%s", buffer.str);
6861 __kmp_str_buf_free(&buffer);
6863 __kmp_env_free(&val);
6866 __kmp_threads_capacity =
6867 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6869 __kmp_tp_capacity = __kmp_default_tp_capacity(
6870 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
6875 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6876 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6877 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6878 __kmp_thread_pool = NULL;
6879 __kmp_thread_pool_insert_pt = NULL;
6880 __kmp_team_pool = NULL;
6887 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6889 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6890 __kmp_root = (kmp_root_t **)((
char *)__kmp_threads +
6891 sizeof(kmp_info_t *) * __kmp_threads_capacity);
6894 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6896 KMP_DEBUG_ASSERT(__kmp_nth == 0);
6901 gtid = __kmp_register_root(TRUE);
6902 KA_TRACE(10, (
"__kmp_do_serial_initialize T#%d\n", gtid));
6903 KMP_ASSERT(KMP_UBER_GTID(gtid));
6904 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
6908 __kmp_common_initialize();
6912 __kmp_register_atfork();
6915 #if !KMP_DYNAMIC_LIB 6919 int rc = atexit(__kmp_internal_end_atexit);
6921 __kmp_fatal(KMP_MSG(FunctionError,
"atexit()"), KMP_ERR(rc),
6927 #if KMP_HANDLE_SIGNALS 6933 __kmp_install_signals(FALSE);
6936 __kmp_install_signals(TRUE);
6941 __kmp_init_counter++;
6943 __kmp_init_serial = TRUE;
6945 if (__kmp_settings) {
6949 if (__kmp_display_env || __kmp_display_env_verbose) {
6950 __kmp_env_print_2();
6959 KA_TRACE(10, (
"__kmp_do_serial_initialize: exit\n"));
6962 void __kmp_serial_initialize(
void) {
6963 if (__kmp_init_serial) {
6966 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6967 if (__kmp_init_serial) {
6968 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6971 __kmp_do_serial_initialize();
6972 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6975 static void __kmp_do_middle_initialize(
void) {
6977 int prev_dflt_team_nth;
6979 if (!__kmp_init_serial) {
6980 __kmp_do_serial_initialize();
6983 KA_TRACE(10, (
"__kmp_middle_initialize: enter\n"));
6987 prev_dflt_team_nth = __kmp_dflt_team_nth;
6989 #if KMP_AFFINITY_SUPPORTED 6992 __kmp_affinity_initialize();
6996 for (i = 0; i < __kmp_threads_capacity; i++) {
6997 if (TCR_PTR(__kmp_threads[i]) != NULL) {
6998 __kmp_affinity_set_init_mask(i, TRUE);
7003 KMP_ASSERT(__kmp_xproc > 0);
7004 if (__kmp_avail_proc == 0) {
7005 __kmp_avail_proc = __kmp_xproc;
7011 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
7012 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
7017 if (__kmp_dflt_team_nth == 0) {
7018 #ifdef KMP_DFLT_NTH_CORES 7020 __kmp_dflt_team_nth = __kmp_ncores;
7021 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 7022 "__kmp_ncores (%d)\n",
7023 __kmp_dflt_team_nth));
7026 __kmp_dflt_team_nth = __kmp_avail_proc;
7027 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = " 7028 "__kmp_avail_proc(%d)\n",
7029 __kmp_dflt_team_nth));
7033 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
7034 __kmp_dflt_team_nth = KMP_MIN_NTH;
7036 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
7037 __kmp_dflt_team_nth = __kmp_sys_max_nth;
7042 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
7044 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
7049 for (i = 0; i < __kmp_threads_capacity; i++) {
7050 kmp_info_t *thread = __kmp_threads[i];
7053 if (thread->th.th_current_task->td_icvs.nproc != 0)
7056 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
7061 (
"__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
7062 __kmp_dflt_team_nth));
7064 #ifdef KMP_ADJUST_BLOCKTIME 7066 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
7067 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
7068 if (__kmp_nth > __kmp_avail_proc) {
7069 __kmp_zero_bt = TRUE;
7075 TCW_SYNC_4(__kmp_init_middle, TRUE);
7077 KA_TRACE(10, (
"__kmp_do_middle_initialize: exit\n"));
7080 void __kmp_middle_initialize(
void) {
7081 if (__kmp_init_middle) {
7084 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7085 if (__kmp_init_middle) {
7086 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7089 __kmp_do_middle_initialize();
7090 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7093 void __kmp_parallel_initialize(
void) {
7094 int gtid = __kmp_entry_gtid();
7097 if (TCR_4(__kmp_init_parallel))
7099 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7100 if (TCR_4(__kmp_init_parallel)) {
7101 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7106 if (TCR_4(__kmp_global.g.g_done)) {
7109 (
"__kmp_parallel_initialize: attempt to init while shutting down\n"));
7110 __kmp_infinite_loop();
7116 if (!__kmp_init_middle) {
7117 __kmp_do_middle_initialize();
7119 __kmp_resume_if_hard_paused();
7122 KA_TRACE(10, (
"__kmp_parallel_initialize: enter\n"));
7123 KMP_ASSERT(KMP_UBER_GTID(gtid));
7125 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 7128 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
7129 __kmp_store_mxcsr(&__kmp_init_mxcsr);
7130 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
7134 #if KMP_HANDLE_SIGNALS 7136 __kmp_install_signals(TRUE);
7140 __kmp_suspend_initialize();
7142 #if defined(USE_LOAD_BALANCE) 7143 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7144 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
7147 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7148 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7152 if (__kmp_version) {
7153 __kmp_print_version_2();
7157 TCW_SYNC_4(__kmp_init_parallel, TRUE);
7160 KA_TRACE(10, (
"__kmp_parallel_initialize: exit\n"));
7162 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7165 void __kmp_hidden_helper_initialize() {
7166 if (TCR_4(__kmp_init_hidden_helper))
7170 if (!TCR_4(__kmp_init_parallel))
7171 __kmp_parallel_initialize();
7175 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7176 if (TCR_4(__kmp_init_hidden_helper)) {
7177 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7182 KMP_ATOMIC_ST_REL(&__kmp_unexecuted_hidden_helper_tasks, 0);
7186 TCW_SYNC_4(__kmp_init_hidden_helper_threads, TRUE);
7189 __kmp_do_initialize_hidden_helper_threads();
7192 __kmp_hidden_helper_threads_initz_wait();
7195 TCW_SYNC_4(__kmp_init_hidden_helper, TRUE);
7197 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7202 void __kmp_run_before_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7204 kmp_disp_t *dispatch;
7209 this_thr->th.th_local.this_construct = 0;
7210 #if KMP_CACHE_MANAGE 7211 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
7213 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
7214 KMP_DEBUG_ASSERT(dispatch);
7215 KMP_DEBUG_ASSERT(team->t.t_dispatch);
7219 dispatch->th_disp_index = 0;
7220 dispatch->th_doacross_buf_idx = 0;
7221 if (__kmp_env_consistency_check)
7222 __kmp_push_parallel(gtid, team->t.t_ident);
7227 void __kmp_run_after_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7229 if (__kmp_env_consistency_check)
7230 __kmp_pop_parallel(gtid, team->t.t_ident);
7232 __kmp_finish_implicit_task(this_thr);
7235 int __kmp_invoke_task_func(
int gtid) {
7237 int tid = __kmp_tid_from_gtid(gtid);
7238 kmp_info_t *this_thr = __kmp_threads[gtid];
7239 kmp_team_t *team = this_thr->th.th_team;
7241 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
7243 if (__itt_stack_caller_create_ptr) {
7245 if (team->t.t_stack_id != NULL) {
7246 __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id);
7248 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL);
7249 __kmp_itt_stack_callee_enter(
7250 (__itt_caller)team->t.t_parent->t.t_stack_id);
7254 #if INCLUDE_SSC_MARKS 7255 SSC_MARK_INVOKING();
7260 void **exit_frame_p;
7261 ompt_data_t *my_task_data;
7262 ompt_data_t *my_parallel_data;
7265 if (ompt_enabled.enabled) {
7266 exit_frame_p = &(team->t.t_implicit_task_taskdata[tid]
7267 .ompt_task_info.frame.exit_frame.ptr);
7269 exit_frame_p = &dummy;
7273 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
7274 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
7275 if (ompt_enabled.ompt_callback_implicit_task) {
7276 ompt_team_size = team->t.t_nproc;
7277 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7278 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
7279 __kmp_tid_from_gtid(gtid), ompt_task_implicit);
7280 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
7284 #if KMP_STATS_ENABLED 7286 if (previous_state == stats_state_e::TEAMS_REGION) {
7287 KMP_PUSH_PARTITIONED_TIMER(OMP_teams);
7289 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel);
7291 KMP_SET_THREAD_STATE(IMPLICIT_TASK);
7294 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
7295 tid, (
int)team->t.t_argc, (
void **)team->t.t_argv
7302 *exit_frame_p = NULL;
7303 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team;
7306 #if KMP_STATS_ENABLED 7307 if (previous_state == stats_state_e::TEAMS_REGION) {
7308 KMP_SET_THREAD_STATE(previous_state);
7310 KMP_POP_PARTITIONED_TIMER();
7314 if (__itt_stack_caller_create_ptr) {
7316 if (team->t.t_stack_id != NULL) {
7317 __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id);
7319 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL);
7320 __kmp_itt_stack_callee_leave(
7321 (__itt_caller)team->t.t_parent->t.t_stack_id);
7325 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
7330 void __kmp_teams_master(
int gtid) {
7332 kmp_info_t *thr = __kmp_threads[gtid];
7333 kmp_team_t *team = thr->th.th_team;
7334 ident_t *loc = team->t.t_ident;
7335 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7336 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7337 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7338 KA_TRACE(20, (
"__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7339 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
7342 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
7345 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
7346 tmp->cg_nthreads = 1;
7347 KA_TRACE(100, (
"__kmp_teams_master: Thread %p created node %p and init" 7348 " cg_nthreads to 1\n",
7350 tmp->up = thr->th.th_cg_roots;
7351 thr->th.th_cg_roots = tmp;
7355 #if INCLUDE_SSC_MARKS 7358 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
7359 (microtask_t)thr->th.th_teams_microtask,
7360 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
7361 #if INCLUDE_SSC_MARKS 7365 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)
7366 thr->th.th_teams_size.nth = thr->th.th_team_nproc;
7369 __kmp_join_call(loc, gtid
7378 int __kmp_invoke_teams_master(
int gtid) {
7379 kmp_info_t *this_thr = __kmp_threads[gtid];
7380 kmp_team_t *team = this_thr->th.th_team;
7382 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7383 KMP_DEBUG_ASSERT((
void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7384 (
void *)__kmp_teams_master);
7386 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7388 int tid = __kmp_tid_from_gtid(gtid);
7389 ompt_data_t *task_data =
7390 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data;
7391 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data;
7392 if (ompt_enabled.ompt_callback_implicit_task) {
7393 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7394 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid,
7396 OMPT_CUR_TASK_INFO(this_thr)->thread_num = tid;
7399 __kmp_teams_master(gtid);
7401 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league;
7403 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7412 void __kmp_push_num_threads(
ident_t *
id,
int gtid,
int num_threads) {
7413 kmp_info_t *thr = __kmp_threads[gtid];
7415 if (num_threads > 0)
7416 thr->th.th_set_nproc = num_threads;
7419 static void __kmp_push_thread_limit(kmp_info_t *thr,
int num_teams,
7421 KMP_DEBUG_ASSERT(thr);
7423 if (!TCR_4(__kmp_init_middle))
7424 __kmp_middle_initialize();
7425 KMP_DEBUG_ASSERT(__kmp_avail_proc);
7426 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth);
7428 if (num_threads == 0) {
7429 if (__kmp_teams_thread_limit > 0) {
7430 num_threads = __kmp_teams_thread_limit;
7432 num_threads = __kmp_avail_proc / num_teams;
7437 if (num_threads > __kmp_dflt_team_nth) {
7438 num_threads = __kmp_dflt_team_nth;
7440 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) {
7441 num_threads = thr->th.th_current_task->td_icvs.thread_limit;
7443 if (num_teams * num_threads > __kmp_teams_max_nth) {
7444 num_threads = __kmp_teams_max_nth / num_teams;
7446 if (num_threads == 0) {
7452 thr->th.th_current_task->td_icvs.thread_limit = num_threads;
7454 if (num_threads > __kmp_dflt_team_nth) {
7455 num_threads = __kmp_dflt_team_nth;
7457 if (num_teams * num_threads > __kmp_teams_max_nth) {
7458 int new_threads = __kmp_teams_max_nth / num_teams;
7459 if (new_threads == 0) {
7462 if (new_threads != num_threads) {
7463 if (!__kmp_reserve_warn) {
7464 __kmp_reserve_warn = 1;
7465 __kmp_msg(kmp_ms_warning,
7466 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7467 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7470 num_threads = new_threads;
7473 thr->th.th_teams_size.nth = num_threads;
7478 void __kmp_push_num_teams(
ident_t *
id,
int gtid,
int num_teams,
7480 kmp_info_t *thr = __kmp_threads[gtid];
7481 KMP_DEBUG_ASSERT(num_teams >= 0);
7482 KMP_DEBUG_ASSERT(num_threads >= 0);
7484 if (num_teams == 0) {
7485 if (__kmp_nteams > 0) {
7486 num_teams = __kmp_nteams;
7491 if (num_teams > __kmp_teams_max_nth) {
7492 if (!__kmp_reserve_warn) {
7493 __kmp_reserve_warn = 1;
7494 __kmp_msg(kmp_ms_warning,
7495 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7496 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7498 num_teams = __kmp_teams_max_nth;
7502 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7504 __kmp_push_thread_limit(thr, num_teams, num_threads);
7509 void __kmp_push_num_teams_51(
ident_t *
id,
int gtid,
int num_teams_lb,
7510 int num_teams_ub,
int num_threads) {
7511 kmp_info_t *thr = __kmp_threads[gtid];
7512 KMP_DEBUG_ASSERT(num_teams_lb >= 0 && num_teams_ub >= 0);
7513 KMP_DEBUG_ASSERT(num_teams_ub >= num_teams_lb);
7514 KMP_DEBUG_ASSERT(num_threads >= 0);
7516 if (num_teams_lb > num_teams_ub) {
7517 __kmp_fatal(KMP_MSG(FailedToCreateTeam, num_teams_lb, num_teams_ub),
7518 KMP_HNT(SetNewBound, __kmp_teams_max_nth), __kmp_msg_null);
7523 if (num_teams_lb == 0 && num_teams_ub > 0)
7524 num_teams_lb = num_teams_ub;
7526 if (num_teams_lb == 0 && num_teams_ub == 0) {
7527 num_teams = (__kmp_nteams > 0) ? __kmp_nteams : num_teams;
7528 if (num_teams > __kmp_teams_max_nth) {
7529 if (!__kmp_reserve_warn) {
7530 __kmp_reserve_warn = 1;
7531 __kmp_msg(kmp_ms_warning,
7532 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7533 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7535 num_teams = __kmp_teams_max_nth;
7537 }
else if (num_teams_lb == num_teams_ub) {
7538 num_teams = num_teams_ub;
7540 if (num_threads == 0) {
7541 if (num_teams_ub > __kmp_teams_max_nth) {
7542 num_teams = num_teams_lb;
7544 num_teams = num_teams_ub;
7547 num_teams = (num_threads > __kmp_teams_max_nth)
7549 : __kmp_teams_max_nth / num_threads;
7550 if (num_teams < num_teams_lb) {
7551 num_teams = num_teams_lb;
7552 }
else if (num_teams > num_teams_ub) {
7553 num_teams = num_teams_ub;
7559 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7561 __kmp_push_thread_limit(thr, num_teams, num_threads);
7565 void __kmp_push_proc_bind(
ident_t *
id,
int gtid, kmp_proc_bind_t proc_bind) {
7566 kmp_info_t *thr = __kmp_threads[gtid];
7567 thr->th.th_set_proc_bind = proc_bind;
7572 void __kmp_internal_fork(
ident_t *
id,
int gtid, kmp_team_t *team) {
7573 kmp_info_t *this_thr = __kmp_threads[gtid];
7579 KMP_DEBUG_ASSERT(team);
7580 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7581 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7584 team->t.t_construct = 0;
7585 team->t.t_ordered.dt.t_value =
7589 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7590 if (team->t.t_max_nproc > 1) {
7592 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7593 team->t.t_disp_buffer[i].buffer_index = i;
7594 team->t.t_disp_buffer[i].doacross_buf_idx = i;
7597 team->t.t_disp_buffer[0].buffer_index = 0;
7598 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7602 KMP_ASSERT(this_thr->th.th_team == team);
7605 for (f = 0; f < team->t.t_nproc; f++) {
7606 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7607 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7612 __kmp_fork_barrier(gtid, 0);
7615 void __kmp_internal_join(
ident_t *
id,
int gtid, kmp_team_t *team) {
7616 kmp_info_t *this_thr = __kmp_threads[gtid];
7618 KMP_DEBUG_ASSERT(team);
7619 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7620 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7626 if (__kmp_threads[gtid] &&
7627 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7628 __kmp_printf(
"GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7629 __kmp_threads[gtid]);
7630 __kmp_printf(
"__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, " 7631 "team->t.t_nproc=%d\n",
7632 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7634 __kmp_print_structure();
7636 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7637 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
7640 __kmp_join_barrier(gtid);
7642 if (ompt_enabled.enabled &&
7643 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
7644 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7645 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
7646 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
7648 void *codeptr = NULL;
7649 if (KMP_MASTER_TID(ds_tid) &&
7650 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7651 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7652 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7654 if (ompt_enabled.ompt_callback_sync_region_wait) {
7655 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
7656 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7659 if (ompt_enabled.ompt_callback_sync_region) {
7660 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
7661 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7665 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7666 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7667 ompt_scope_end, NULL, task_data, 0, ds_tid,
7668 ompt_task_implicit);
7674 KMP_ASSERT(this_thr->th.th_team == team);
7679 #ifdef USE_LOAD_BALANCE 7683 static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7686 kmp_team_t *hot_team;
7688 if (root->r.r_active) {
7691 hot_team = root->r.r_hot_team;
7692 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7693 return hot_team->t.t_nproc - 1;
7698 for (i = 1; i < hot_team->t.t_nproc; i++) {
7699 if (hot_team->t.t_threads[i]->th.th_active) {
7708 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc) {
7711 int hot_team_active;
7712 int team_curr_active;
7715 KB_TRACE(20, (
"__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7717 KMP_DEBUG_ASSERT(root);
7718 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7719 ->th.th_current_task->td_icvs.dynamic == TRUE);
7720 KMP_DEBUG_ASSERT(set_nproc > 1);
7722 if (set_nproc == 1) {
7723 KB_TRACE(20, (
"__kmp_load_balance_nproc: serial execution.\n"));
7732 pool_active = __kmp_thread_pool_active_nth;
7733 hot_team_active = __kmp_active_hot_team_nproc(root);
7734 team_curr_active = pool_active + hot_team_active + 1;
7737 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7738 KB_TRACE(30, (
"__kmp_load_balance_nproc: system active = %d pool active = %d " 7739 "hot team active = %d\n",
7740 system_active, pool_active, hot_team_active));
7742 if (system_active < 0) {
7746 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7747 KMP_WARNING(CantLoadBalUsing,
"KMP_DYNAMIC_MODE=thread limit");
7750 retval = __kmp_avail_proc - __kmp_nth +
7751 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7752 if (retval > set_nproc) {
7755 if (retval < KMP_MIN_NTH) {
7756 retval = KMP_MIN_NTH;
7759 KB_TRACE(20, (
"__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7767 if (system_active < team_curr_active) {
7768 system_active = team_curr_active;
7770 retval = __kmp_avail_proc - system_active + team_curr_active;
7771 if (retval > set_nproc) {
7774 if (retval < KMP_MIN_NTH) {
7775 retval = KMP_MIN_NTH;
7778 KB_TRACE(20, (
"__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7787 void __kmp_cleanup(
void) {
7790 KA_TRACE(10, (
"__kmp_cleanup: enter\n"));
7792 if (TCR_4(__kmp_init_parallel)) {
7793 #if KMP_HANDLE_SIGNALS 7794 __kmp_remove_signals();
7796 TCW_4(__kmp_init_parallel, FALSE);
7799 if (TCR_4(__kmp_init_middle)) {
7800 #if KMP_AFFINITY_SUPPORTED 7801 __kmp_affinity_uninitialize();
7803 __kmp_cleanup_hierarchy();
7804 TCW_4(__kmp_init_middle, FALSE);
7807 KA_TRACE(10, (
"__kmp_cleanup: go serial cleanup\n"));
7809 if (__kmp_init_serial) {
7810 __kmp_runtime_destroy();
7811 __kmp_init_serial = FALSE;
7814 __kmp_cleanup_threadprivate_caches();
7816 for (f = 0; f < __kmp_threads_capacity; f++) {
7817 if (__kmp_root[f] != NULL) {
7818 __kmp_free(__kmp_root[f]);
7819 __kmp_root[f] = NULL;
7822 __kmp_free(__kmp_threads);
7825 __kmp_threads = NULL;
7827 __kmp_threads_capacity = 0;
7829 #if KMP_USE_DYNAMIC_LOCK 7830 __kmp_cleanup_indirect_user_locks();
7832 __kmp_cleanup_user_locks();
7835 #if KMP_AFFINITY_SUPPORTED 7836 KMP_INTERNAL_FREE(CCAST(
char *, __kmp_cpuinfo_file));
7837 __kmp_cpuinfo_file = NULL;
7840 #if KMP_USE_ADAPTIVE_LOCKS 7841 #if KMP_DEBUG_ADAPTIVE_LOCKS 7842 __kmp_print_speculative_stats();
7845 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7846 __kmp_nested_nth.nth = NULL;
7847 __kmp_nested_nth.size = 0;
7848 __kmp_nested_nth.used = 0;
7849 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7850 __kmp_nested_proc_bind.bind_types = NULL;
7851 __kmp_nested_proc_bind.size = 0;
7852 __kmp_nested_proc_bind.used = 0;
7853 if (__kmp_affinity_format) {
7854 KMP_INTERNAL_FREE(__kmp_affinity_format);
7855 __kmp_affinity_format = NULL;
7858 __kmp_i18n_catclose();
7860 #if KMP_USE_HIER_SCHED 7861 __kmp_hier_scheds.deallocate();
7864 #if KMP_STATS_ENABLED 7868 KA_TRACE(10, (
"__kmp_cleanup: exit\n"));
7873 int __kmp_ignore_mppbeg(
void) {
7876 if ((env = getenv(
"KMP_IGNORE_MPPBEG")) != NULL) {
7877 if (__kmp_str_match_false(env))
7884 int __kmp_ignore_mppend(
void) {
7887 if ((env = getenv(
"KMP_IGNORE_MPPEND")) != NULL) {
7888 if (__kmp_str_match_false(env))
7895 void __kmp_internal_begin(
void) {
7901 gtid = __kmp_entry_gtid();
7902 root = __kmp_threads[gtid]->th.th_root;
7903 KMP_ASSERT(KMP_UBER_GTID(gtid));
7905 if (root->r.r_begin)
7907 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7908 if (root->r.r_begin) {
7909 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7913 root->r.r_begin = TRUE;
7915 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7920 void __kmp_user_set_library(
enum library_type arg) {
7927 gtid = __kmp_entry_gtid();
7928 thread = __kmp_threads[gtid];
7930 root = thread->th.th_root;
7932 KA_TRACE(20, (
"__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7934 if (root->r.r_in_parallel) {
7936 KMP_WARNING(SetLibraryIncorrectCall);
7941 case library_serial:
7942 thread->th.th_set_nproc = 0;
7943 set__nproc(thread, 1);
7945 case library_turnaround:
7946 thread->th.th_set_nproc = 0;
7947 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7948 : __kmp_dflt_team_nth_ub);
7950 case library_throughput:
7951 thread->th.th_set_nproc = 0;
7952 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7953 : __kmp_dflt_team_nth_ub);
7956 KMP_FATAL(UnknownLibraryType, arg);
7959 __kmp_aux_set_library(arg);
7962 void __kmp_aux_set_stacksize(
size_t arg) {
7963 if (!__kmp_init_serial)
7964 __kmp_serial_initialize();
7967 if (arg & (0x1000 - 1)) {
7968 arg &= ~(0x1000 - 1);
7973 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7976 if (!TCR_4(__kmp_init_parallel)) {
7979 if (value < __kmp_sys_min_stksize)
7980 value = __kmp_sys_min_stksize;
7981 else if (value > KMP_MAX_STKSIZE)
7982 value = KMP_MAX_STKSIZE;
7984 __kmp_stksize = value;
7986 __kmp_env_stksize = TRUE;
7989 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7994 void __kmp_aux_set_library(
enum library_type arg) {
7995 __kmp_library = arg;
7997 switch (__kmp_library) {
7998 case library_serial: {
7999 KMP_INFORM(LibraryIsSerial);
8001 case library_turnaround:
8002 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set)
8003 __kmp_use_yield = 2;
8005 case library_throughput:
8006 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
8007 __kmp_dflt_blocktime = 200;
8010 KMP_FATAL(UnknownLibraryType, arg);
8016 static kmp_team_t *__kmp_aux_get_team_info(
int &teams_serialized) {
8017 kmp_info_t *thr = __kmp_entry_thread();
8018 teams_serialized = 0;
8019 if (thr->th.th_teams_microtask) {
8020 kmp_team_t *team = thr->th.th_team;
8021 int tlevel = thr->th.th_teams_level;
8022 int ii = team->t.t_level;
8023 teams_serialized = team->t.t_serialized;
8024 int level = tlevel + 1;
8025 KMP_DEBUG_ASSERT(ii >= tlevel);
8026 while (ii > level) {
8027 for (teams_serialized = team->t.t_serialized;
8028 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
8030 if (team->t.t_serialized && (!teams_serialized)) {
8031 team = team->t.t_parent;
8035 team = team->t.t_parent;
8044 int __kmp_aux_get_team_num() {
8046 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
8048 if (serialized > 1) {
8051 return team->t.t_master_tid;
8057 int __kmp_aux_get_num_teams() {
8059 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
8061 if (serialized > 1) {
8064 return team->t.t_parent->t.t_nproc;
8103 typedef struct kmp_affinity_format_field_t {
8105 const char *long_name;
8108 } kmp_affinity_format_field_t;
8110 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
8111 #if KMP_AFFINITY_SUPPORTED 8112 {
'A',
"thread_affinity",
's'},
8114 {
't',
"team_num",
'd'},
8115 {
'T',
"num_teams",
'd'},
8116 {
'L',
"nesting_level",
'd'},
8117 {
'n',
"thread_num",
'd'},
8118 {
'N',
"num_threads",
'd'},
8119 {
'a',
"ancestor_tnum",
'd'},
8121 {
'P',
"process_id",
'd'},
8122 {
'i',
"native_thread_id",
'd'}};
8125 static int __kmp_aux_capture_affinity_field(
int gtid,
const kmp_info_t *th,
8127 kmp_str_buf_t *field_buffer) {
8128 int rc, format_index, field_value;
8129 const char *width_left, *width_right;
8130 bool pad_zeros, right_justify, parse_long_name, found_valid_name;
8131 static const int FORMAT_SIZE = 20;
8132 char format[FORMAT_SIZE] = {0};
8133 char absolute_short_name = 0;
8135 KMP_DEBUG_ASSERT(gtid >= 0);
8136 KMP_DEBUG_ASSERT(th);
8137 KMP_DEBUG_ASSERT(**ptr ==
'%');
8138 KMP_DEBUG_ASSERT(field_buffer);
8140 __kmp_str_buf_clear(field_buffer);
8147 __kmp_str_buf_cat(field_buffer,
"%", 1);
8158 right_justify =
false;
8160 right_justify =
true;
8164 width_left = width_right = NULL;
8165 if (**ptr >=
'0' && **ptr <=
'9') {
8173 format[format_index++] =
'%';
8175 format[format_index++] =
'-';
8177 format[format_index++] =
'0';
8178 if (width_left && width_right) {
8182 while (i < 8 && width_left < width_right) {
8183 format[format_index++] = *width_left;
8191 found_valid_name =
false;
8192 parse_long_name = (**ptr ==
'{');
8193 if (parse_long_name)
8195 for (
size_t i = 0; i <
sizeof(__kmp_affinity_format_table) /
8196 sizeof(__kmp_affinity_format_table[0]);
8198 char short_name = __kmp_affinity_format_table[i].short_name;
8199 const char *long_name = __kmp_affinity_format_table[i].long_name;
8200 char field_format = __kmp_affinity_format_table[i].field_format;
8201 if (parse_long_name) {
8202 size_t length = KMP_STRLEN(long_name);
8203 if (strncmp(*ptr, long_name, length) == 0) {
8204 found_valid_name =
true;
8207 }
else if (**ptr == short_name) {
8208 found_valid_name =
true;
8211 if (found_valid_name) {
8212 format[format_index++] = field_format;
8213 format[format_index++] =
'\0';
8214 absolute_short_name = short_name;
8218 if (parse_long_name) {
8220 absolute_short_name = 0;
8228 switch (absolute_short_name) {
8230 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
8233 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
8236 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
8239 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
8242 static const int BUFFER_SIZE = 256;
8243 char buf[BUFFER_SIZE];
8244 __kmp_expand_host_name(buf, BUFFER_SIZE);
8245 rc = __kmp_str_buf_print(field_buffer, format, buf);
8248 rc = __kmp_str_buf_print(field_buffer, format, getpid());
8251 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid());
8254 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
8258 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
8259 rc = __kmp_str_buf_print(field_buffer, format, field_value);
8261 #if KMP_AFFINITY_SUPPORTED 8264 __kmp_str_buf_init(&buf);
8265 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
8266 rc = __kmp_str_buf_print(field_buffer, format, buf.str);
8267 __kmp_str_buf_free(&buf);
8273 rc = __kmp_str_buf_print(field_buffer,
"%s",
"undefined");
8275 if (parse_long_name) {
8284 KMP_ASSERT(format_index <= FORMAT_SIZE);
8294 size_t __kmp_aux_capture_affinity(
int gtid,
const char *format,
8295 kmp_str_buf_t *buffer) {
8296 const char *parse_ptr;
8298 const kmp_info_t *th;
8299 kmp_str_buf_t field;
8301 KMP_DEBUG_ASSERT(buffer);
8302 KMP_DEBUG_ASSERT(gtid >= 0);
8304 __kmp_str_buf_init(&field);
8305 __kmp_str_buf_clear(buffer);
8307 th = __kmp_threads[gtid];
8313 if (parse_ptr == NULL || *parse_ptr ==
'\0') {
8314 parse_ptr = __kmp_affinity_format;
8316 KMP_DEBUG_ASSERT(parse_ptr);
8318 while (*parse_ptr !=
'\0') {
8320 if (*parse_ptr ==
'%') {
8322 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
8323 __kmp_str_buf_catbuf(buffer, &field);
8327 __kmp_str_buf_cat(buffer, parse_ptr, 1);
8332 __kmp_str_buf_free(&field);
8337 void __kmp_aux_display_affinity(
int gtid,
const char *format) {
8339 __kmp_str_buf_init(&buf);
8340 __kmp_aux_capture_affinity(gtid, format, &buf);
8341 __kmp_fprintf(kmp_out,
"%s" KMP_END_OF_LINE, buf.str);
8342 __kmp_str_buf_free(&buf);
8347 void __kmp_aux_set_blocktime(
int arg, kmp_info_t *thread,
int tid) {
8348 int blocktime = arg;
8354 __kmp_save_internal_controls(thread);
8357 if (blocktime < KMP_MIN_BLOCKTIME)
8358 blocktime = KMP_MIN_BLOCKTIME;
8359 else if (blocktime > KMP_MAX_BLOCKTIME)
8360 blocktime = KMP_MAX_BLOCKTIME;
8362 set__blocktime_team(thread->th.th_team, tid, blocktime);
8363 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
8367 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
8369 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
8370 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
8376 set__bt_set_team(thread->th.th_team, tid, bt_set);
8377 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
8379 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, " 8380 "bt_intervals=%d, monitor_updates=%d\n",
8381 __kmp_gtid_from_tid(tid, thread->th.th_team),
8382 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
8383 __kmp_monitor_wakeups));
8385 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
8386 __kmp_gtid_from_tid(tid, thread->th.th_team),
8387 thread->th.th_team->t.t_id, tid, blocktime));
8391 void __kmp_aux_set_defaults(
char const *str,
size_t len) {
8392 if (!__kmp_init_serial) {
8393 __kmp_serial_initialize();
8395 __kmp_env_initialize(str);
8397 if (__kmp_settings || __kmp_display_env || __kmp_display_env_verbose) {
8405 PACKED_REDUCTION_METHOD_T
8406 __kmp_determine_reduction_method(
8407 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
size_t reduce_size,
8408 void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
8409 kmp_critical_name *lck) {
8420 PACKED_REDUCTION_METHOD_T retval;
8424 KMP_DEBUG_ASSERT(loc);
8425 KMP_DEBUG_ASSERT(lck);
8427 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \ 8428 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE)) 8429 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func)) 8431 retval = critical_reduce_block;
8434 team_size = __kmp_get_team_num_threads(global_tid);
8435 if (team_size == 1) {
8437 retval = empty_reduce_block;
8441 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8443 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \ 8444 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 8446 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 8447 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD 8449 int teamsize_cutoff = 4;
8451 #if KMP_MIC_SUPPORTED 8452 if (__kmp_mic_type != non_mic) {
8453 teamsize_cutoff = 8;
8456 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8457 if (tree_available) {
8458 if (team_size <= teamsize_cutoff) {
8459 if (atomic_available) {
8460 retval = atomic_reduce_block;
8463 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8465 }
else if (atomic_available) {
8466 retval = atomic_reduce_block;
8469 #error "Unknown or unsupported OS" 8470 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || 8473 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS 8475 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_HURD 8479 if (atomic_available) {
8480 if (num_vars <= 2) {
8481 retval = atomic_reduce_block;
8487 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8488 if (atomic_available && (num_vars <= 3)) {
8489 retval = atomic_reduce_block;
8490 }
else if (tree_available) {
8491 if ((reduce_size > (9 *
sizeof(kmp_real64))) &&
8492 (reduce_size < (2000 *
sizeof(kmp_real64)))) {
8493 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
8498 #error "Unknown or unsupported OS" 8502 #error "Unknown or unsupported architecture" 8510 if (__kmp_force_reduction_method != reduction_method_not_defined &&
8513 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
8515 int atomic_available, tree_available;
8517 switch ((forced_retval = __kmp_force_reduction_method)) {
8518 case critical_reduce_block:
8522 case atomic_reduce_block:
8523 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8524 if (!atomic_available) {
8525 KMP_WARNING(RedMethodNotSupported,
"atomic");
8526 forced_retval = critical_reduce_block;
8530 case tree_reduce_block:
8531 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8532 if (!tree_available) {
8533 KMP_WARNING(RedMethodNotSupported,
"tree");
8534 forced_retval = critical_reduce_block;
8536 #if KMP_FAST_REDUCTION_BARRIER 8537 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8546 retval = forced_retval;
8549 KA_TRACE(10, (
"reduction method selected=%08x\n", retval));
8551 #undef FAST_REDUCTION_TREE_METHOD_GENERATED 8552 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED 8557 kmp_int32 __kmp_get_reduce_method(
void) {
8558 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
8563 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
8567 void __kmp_hard_pause() {
8568 __kmp_pause_status = kmp_hard_paused;
8569 __kmp_internal_end_thread(-1);
8573 void __kmp_resume_if_soft_paused() {
8574 if (__kmp_pause_status == kmp_soft_paused) {
8575 __kmp_pause_status = kmp_not_paused;
8577 for (
int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
8578 kmp_info_t *thread = __kmp_threads[gtid];
8580 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
8582 if (fl.is_sleeping())
8584 else if (__kmp_try_suspend_mx(thread)) {
8585 __kmp_unlock_suspend_mx(thread);
8588 if (fl.is_sleeping()) {
8591 }
else if (__kmp_try_suspend_mx(thread)) {
8592 __kmp_unlock_suspend_mx(thread);
8604 int __kmp_pause_resource(kmp_pause_status_t level) {
8605 if (level == kmp_not_paused) {
8606 if (__kmp_pause_status == kmp_not_paused) {
8610 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
8611 __kmp_pause_status == kmp_hard_paused);
8612 __kmp_pause_status = kmp_not_paused;
8615 }
else if (level == kmp_soft_paused) {
8616 if (__kmp_pause_status != kmp_not_paused) {
8623 }
else if (level == kmp_hard_paused) {
8624 if (__kmp_pause_status != kmp_not_paused) {
8637 void __kmp_omp_display_env(
int verbose) {
8638 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
8639 if (__kmp_init_serial == 0)
8640 __kmp_do_serial_initialize();
8641 __kmp_display_env_impl(!verbose, verbose);
8642 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
8646 kmp_info_t **__kmp_hidden_helper_threads;
8647 kmp_info_t *__kmp_hidden_helper_main_thread;
8648 kmp_int32 __kmp_hidden_helper_threads_num = 8;
8649 std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;
8651 kmp_int32 __kmp_enable_hidden_helper = TRUE;
8653 kmp_int32 __kmp_enable_hidden_helper = FALSE;
8657 std::atomic<kmp_int32> __kmp_hit_hidden_helper_threads_num;
8659 void __kmp_hidden_helper_wrapper_fn(
int *gtid,
int *, ...) {
8664 KMP_ATOMIC_INC(&__kmp_hit_hidden_helper_threads_num);
8665 while (KMP_ATOMIC_LD_ACQ(&__kmp_hit_hidden_helper_threads_num) !=
8666 __kmp_hidden_helper_threads_num)
8672 TCW_4(__kmp_init_hidden_helper_threads, FALSE);
8673 __kmp_hidden_helper_initz_release();
8674 __kmp_hidden_helper_main_thread_wait();
8676 for (
int i = 1; i < __kmp_hit_hidden_helper_threads_num; ++i) {
8677 __kmp_hidden_helper_worker_thread_signal();
8683 void __kmp_hidden_helper_threads_initz_routine() {
8685 const int gtid = __kmp_register_root(TRUE);
8686 __kmp_hidden_helper_main_thread = __kmp_threads[gtid];
8687 __kmp_hidden_helper_threads = &__kmp_threads[gtid];
8688 __kmp_hidden_helper_main_thread->th.th_set_nproc =
8689 __kmp_hidden_helper_threads_num;
8691 KMP_ATOMIC_ST_REL(&__kmp_hit_hidden_helper_threads_num, 0);
8696 TCW_SYNC_4(__kmp_init_hidden_helper, FALSE);
8698 __kmp_hidden_helper_threads_deinitz_release();
KMP_EXPORT kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid)
#define KMP_COUNT_VALUE(name, value)
Adds value to specified timer (name).
KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid)
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.
KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid)
stats_state_e
the states which a thread can be in
KMP_EXPORT void __kmpc_fork_call(ident_t *, kmp_int32 nargs, kmpc_micro microtask,...)