36 #include "kmp_wrapper_getpid.h"
41 #include "kmp_stats.h"
42 #include "kmp_wait_release.h"
50 #include <sys/times.h>
51 #include <sys/resource.h>
52 #include <sys/syscall.h>
54 #if KMP_OS_LINUX && !KMP_OS_CNK
55 # include <sys/sysinfo.h>
56 # if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
70 # include <sys/sysctl.h>
71 # include <mach/mach.h>
73 # include <sys/sysctl.h>
74 # include <pthread_np.h>
83 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
91 struct kmp_sys_timer {
92 struct timespec start;
96 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
98 static struct kmp_sys_timer __kmp_sys_timer_data;
100 #if KMP_HANDLE_SIGNALS
101 typedef void (* sig_func_t )( int );
102 STATIC_EFI2_WORKAROUND
struct sigaction __kmp_sighldrs[ NSIG ];
103 static sigset_t __kmp_sigset;
106 static int __kmp_init_runtime = FALSE;
108 static int __kmp_fork_count = 0;
110 static pthread_condattr_t __kmp_suspend_cond_attr;
111 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
113 static kmp_cond_align_t __kmp_wait_cv;
114 static kmp_mutex_align_t __kmp_wait_mx;
121 __kmp_print_cond(
char *buffer, kmp_cond_align_t *cond )
123 KMP_SNPRINTF( buffer, 128,
"(cond (lock (%ld, %d)), (descr (%p)))",
124 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
125 cond->c_cond.__c_waiting );
132 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
145 # if KMP_ARCH_X86 || KMP_ARCH_ARM
146 # ifndef __NR_sched_setaffinity
147 # define __NR_sched_setaffinity 241
148 # elif __NR_sched_setaffinity != 241
149 # error Wrong code for setaffinity system call.
151 # ifndef __NR_sched_getaffinity
152 # define __NR_sched_getaffinity 242
153 # elif __NR_sched_getaffinity != 242
154 # error Wrong code for getaffinity system call.
157 # elif KMP_ARCH_AARCH64
158 # ifndef __NR_sched_setaffinity
159 # define __NR_sched_setaffinity 122
160 # elif __NR_sched_setaffinity != 122
161 # error Wrong code for setaffinity system call.
163 # ifndef __NR_sched_getaffinity
164 # define __NR_sched_getaffinity 123
165 # elif __NR_sched_getaffinity != 123
166 # error Wrong code for getaffinity system call.
169 # elif KMP_ARCH_X86_64
170 # ifndef __NR_sched_setaffinity
171 # define __NR_sched_setaffinity 203
172 # elif __NR_sched_setaffinity != 203
173 # error Wrong code for setaffinity system call.
175 # ifndef __NR_sched_getaffinity
176 # define __NR_sched_getaffinity 204
177 # elif __NR_sched_getaffinity != 204
178 # error Wrong code for getaffinity system call.
181 # elif KMP_ARCH_PPC64
182 # ifndef __NR_sched_setaffinity
183 # define __NR_sched_setaffinity 222
184 # elif __NR_sched_setaffinity != 222
185 # error Wrong code for setaffinity system call.
187 # ifndef __NR_sched_getaffinity
188 # define __NR_sched_getaffinity 223
189 # elif __NR_sched_getaffinity != 223
190 # error Wrong code for getaffinity system call.
195 # error Unknown or unsupported architecture
200 __kmp_set_system_affinity( kmp_affin_mask_t
const *mask,
int abort_on_error )
202 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
203 "Illegal set affinity operation when not capable");
205 int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
210 if (abort_on_error) {
213 KMP_MSG( FatalSysError ),
222 __kmp_get_system_affinity( kmp_affin_mask_t *mask,
int abort_on_error )
224 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
225 "Illegal get affinity operation when not capable");
227 int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
232 if (abort_on_error) {
235 KMP_MSG( FatalSysError ),
244 __kmp_affinity_bind_thread(
int which )
246 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
247 "Illegal set affinity operation when not capable");
249 kmp_affin_mask_t *mask = (kmp_affin_mask_t *)KMP_ALLOCA(__kmp_affin_mask_size);
251 KMP_CPU_SET(which, mask);
252 __kmp_set_system_affinity(mask, TRUE);
261 __kmp_affinity_determine_capable(
const char *env_var)
267 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
271 kmp_affin_mask_t *buf;
272 buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
277 gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
278 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
279 "initial getaffinity call returned %d errno = %d\n",
287 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
288 && (__kmp_affinity_type != affinity_none)
289 && (__kmp_affinity_type != affinity_default)
290 && (__kmp_affinity_type != affinity_disabled))) {
294 KMP_MSG( GetAffSysCallNotSupported, env_var ),
299 KMP_AFFINITY_DISABLE();
300 KMP_INTERNAL_FREE(buf);
309 sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
310 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
311 "setaffinity for mask size %d returned %d errno = %d\n",
312 gCode, sCode, errno));
314 if (errno == ENOSYS) {
315 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
316 && (__kmp_affinity_type != affinity_none)
317 && (__kmp_affinity_type != affinity_default)
318 && (__kmp_affinity_type != affinity_disabled))) {
322 KMP_MSG( SetAffSysCallNotSupported, env_var ),
327 KMP_AFFINITY_DISABLE();
328 KMP_INTERNAL_FREE(buf);
330 if (errno == EFAULT) {
331 KMP_AFFINITY_ENABLE(gCode);
332 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
333 "affinity supported (mask size %d)\n",
334 (
int)__kmp_affin_mask_size));
335 KMP_INTERNAL_FREE(buf);
345 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
346 "searching for proper set size\n"));
348 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
349 gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
350 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
351 "getaffinity for mask size %d returned %d errno = %d\n", size,
355 if ( errno == ENOSYS )
360 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
361 "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
363 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
364 && (__kmp_affinity_type != affinity_none)
365 && (__kmp_affinity_type != affinity_default)
366 && (__kmp_affinity_type != affinity_disabled))) {
370 KMP_MSG( GetAffSysCallNotSupported, env_var ),
375 KMP_AFFINITY_DISABLE();
376 KMP_INTERNAL_FREE(buf);
382 sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
383 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
384 "setaffinity for mask size %d returned %d errno = %d\n",
385 gCode, sCode, errno));
387 if (errno == ENOSYS) {
391 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
392 "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
394 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
395 && (__kmp_affinity_type != affinity_none)
396 && (__kmp_affinity_type != affinity_default)
397 && (__kmp_affinity_type != affinity_disabled))) {
401 KMP_MSG( SetAffSysCallNotSupported, env_var ),
406 KMP_AFFINITY_DISABLE();
407 KMP_INTERNAL_FREE(buf);
410 if (errno == EFAULT) {
411 KMP_AFFINITY_ENABLE(gCode);
412 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
413 "affinity supported (mask size %d)\n",
414 (
int)__kmp_affin_mask_size));
415 KMP_INTERNAL_FREE(buf);
421 KMP_INTERNAL_FREE(buf);
427 KMP_AFFINITY_DISABLE();
428 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
429 "cannot determine mask size - affinity not supported\n"));
430 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
431 && (__kmp_affinity_type != affinity_none)
432 && (__kmp_affinity_type != affinity_default)
433 && (__kmp_affinity_type != affinity_disabled))) {
434 KMP_WARNING( AffCantGetMaskSize, env_var );
438 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
443 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && !KMP_OS_CNK
446 __kmp_futex_determine_capable()
449 int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
450 int retval = ( rc == 0 ) || ( errno != ENOSYS );
452 KA_TRACE(10, (
"__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
454 KA_TRACE(10, (
"__kmp_futex_determine_capable: futex syscall%s supported\n",
455 retval ?
"" :
" not" ) );
460 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
465 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
472 __kmp_test_then_or8(
volatile kmp_int8 *p, kmp_int8 d )
474 kmp_int8 old_value, new_value;
476 old_value = TCR_1( *p );
477 new_value = old_value | d;
479 while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
482 old_value = TCR_1( *p );
483 new_value = old_value | d;
489 __kmp_test_then_and8(
volatile kmp_int8 *p, kmp_int8 d )
491 kmp_int8 old_value, new_value;
493 old_value = TCR_1( *p );
494 new_value = old_value & d;
496 while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
499 old_value = TCR_1( *p );
500 new_value = old_value & d;
506 __kmp_test_then_or32(
volatile kmp_int32 *p, kmp_int32 d )
508 kmp_int32 old_value, new_value;
510 old_value = TCR_4( *p );
511 new_value = old_value | d;
513 while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
516 old_value = TCR_4( *p );
517 new_value = old_value | d;
523 __kmp_test_then_and32(
volatile kmp_int32 *p, kmp_int32 d )
525 kmp_int32 old_value, new_value;
527 old_value = TCR_4( *p );
528 new_value = old_value & d;
530 while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
533 old_value = TCR_4( *p );
534 new_value = old_value & d;
539 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
541 __kmp_test_then_add8(
volatile kmp_int8 *p, kmp_int8 d )
543 kmp_int8 old_value, new_value;
545 old_value = TCR_1( *p );
546 new_value = old_value + d;
548 while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
551 old_value = TCR_1( *p );
552 new_value = old_value + d;
558 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 d )
560 kmp_int64 old_value, new_value;
562 old_value = TCR_8( *p );
563 new_value = old_value + d;
565 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
568 old_value = TCR_8( *p );
569 new_value = old_value + d;
576 __kmp_test_then_or64(
volatile kmp_int64 *p, kmp_int64 d )
578 kmp_int64 old_value, new_value;
580 old_value = TCR_8( *p );
581 new_value = old_value | d;
582 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
585 old_value = TCR_8( *p );
586 new_value = old_value | d;
592 __kmp_test_then_and64(
volatile kmp_int64 *p, kmp_int64 d )
594 kmp_int64 old_value, new_value;
596 old_value = TCR_8( *p );
597 new_value = old_value & d;
598 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
601 old_value = TCR_8( *p );
602 new_value = old_value & d;
610 __kmp_terminate_thread(
int gtid )
613 kmp_info_t *th = __kmp_threads[ gtid ];
617 #ifdef KMP_CANCEL_THREADS
618 KA_TRACE( 10, (
"__kmp_terminate_thread: kill (%d)\n", gtid ) );
619 status = pthread_cancel( th->th.th_info.ds.ds_thread );
620 if ( status != 0 && status != ESRCH ) {
623 KMP_MSG( CantTerminateWorkerThread ),
647 __kmp_set_stack_info(
int gtid, kmp_info_t *th )
650 #if KMP_OS_LINUX || KMP_OS_FREEBSD
660 if ( ! KMP_UBER_GTID(gtid) ) {
663 status = pthread_attr_init( &attr );
664 KMP_CHECK_SYSFAIL(
"pthread_attr_init", status );
666 status = pthread_attr_get_np( pthread_self(), &attr );
667 KMP_CHECK_SYSFAIL(
"pthread_attr_get_np", status );
669 status = pthread_getattr_np( pthread_self(), &attr );
670 KMP_CHECK_SYSFAIL(
"pthread_getattr_np", status );
672 status = pthread_attr_getstack( &attr, &addr, &size );
673 KMP_CHECK_SYSFAIL(
"pthread_attr_getstack", status );
674 KA_TRACE( 60, (
"__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
678 status = pthread_attr_destroy( &attr );
679 KMP_CHECK_SYSFAIL(
"pthread_attr_destroy", status );
682 if ( size != 0 && addr != 0 ) {
684 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((
char *)addr) + size));
685 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
686 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
691 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
692 TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
693 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
698 __kmp_launch_worker(
void *thr )
700 int status, old_type, old_state;
701 #ifdef KMP_BLOCK_SIGNALS
702 sigset_t new_set, old_set;
705 void *
volatile padding = 0;
709 gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
710 __kmp_gtid_set_specific( gtid );
711 #ifdef KMP_TDATA_GTID
714 #if KMP_STATS_ENABLED
716 __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
720 __kmp_itt_thread_name( gtid );
723 #if KMP_AFFINITY_SUPPORTED
724 __kmp_affinity_set_init_mask( gtid, FALSE );
727 #ifdef KMP_CANCEL_THREADS
728 status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
729 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status );
731 status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
732 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
735 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
740 __kmp_clear_x87_fpu_status_word();
741 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
742 __kmp_load_mxcsr( &__kmp_init_mxcsr );
745 #ifdef KMP_BLOCK_SIGNALS
746 status = sigfillset( & new_set );
747 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status );
748 status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
749 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
752 #if KMP_OS_LINUX || KMP_OS_FREEBSD
753 if ( __kmp_stkoffset > 0 && gtid > 0 ) {
754 padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
759 __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
761 __kmp_check_stack_overlap( (kmp_info_t*)thr );
763 exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
765 #ifdef KMP_BLOCK_SIGNALS
766 status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
767 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
777 __kmp_launch_monitor(
void *thr )
779 int status, old_type, old_state;
780 #ifdef KMP_BLOCK_SIGNALS
783 struct timespec interval;
785 int yield_cycles = 0;
790 KA_TRACE( 10, (
"__kmp_launch_monitor: #1 launched\n" ) );
793 __kmp_gtid_set_specific( KMP_GTID_MONITOR );
794 #ifdef KMP_TDATA_GTID
795 __kmp_gtid = KMP_GTID_MONITOR;
801 __kmp_itt_thread_ignore();
804 __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
806 __kmp_check_stack_overlap( (kmp_info_t*)thr );
808 #ifdef KMP_CANCEL_THREADS
809 status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
810 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status );
812 status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
813 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
816 #if KMP_REAL_TIME_FIX
820 int sched = sched_getscheduler( 0 );
821 if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
824 struct sched_param param;
825 int max_priority = sched_get_priority_max( sched );
827 KMP_WARNING( RealTimeSchedNotSupported );
828 sched_getparam( 0, & param );
829 if ( param.sched_priority < max_priority ) {
830 param.sched_priority += 1;
831 rc = sched_setscheduler( 0, sched, & param );
836 KMP_MSG( CantChangeMonitorPriority ),
838 KMP_MSG( MonitorWillStarve ),
847 KMP_MSG( RunningAtMaxPriority ),
848 KMP_MSG( MonitorWillStarve ),
849 KMP_HNT( RunningAtMaxPriority ),
854 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
856 #endif // KMP_REAL_TIME_FIX
860 if ( __kmp_monitor_wakeups == 1 ) {
862 interval.tv_nsec = 0;
865 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
868 KA_TRACE( 10, (
"__kmp_launch_monitor: #2 monitor\n" ) );
870 if (__kmp_yield_cycle) {
871 __kmp_yielding_on = 0;
872 yield_count = __kmp_yield_off_count;
874 __kmp_yielding_on = 1;
877 while( ! TCR_4( __kmp_global.g.g_done ) ) {
883 KA_TRACE( 15, (
"__kmp_launch_monitor: update\n" ) );
885 status = gettimeofday( &tval, NULL );
886 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
887 TIMEVAL_TO_TIMESPEC( &tval, &now );
889 now.tv_sec += interval.tv_sec;
890 now.tv_nsec += interval.tv_nsec;
892 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
894 now.tv_nsec -= KMP_NSEC_PER_SEC;
897 status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
898 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
900 if ( !TCR_4(__kmp_global.g.g_done) ) {
901 status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
903 if ( status != ETIMEDOUT && status != EINTR ) {
904 KMP_SYSFAIL(
"pthread_cond_timedwait", status );
908 status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
909 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
911 if (__kmp_yield_cycle) {
913 if ( (yield_cycles % yield_count) == 0 ) {
914 if (__kmp_yielding_on) {
915 __kmp_yielding_on = 0;
916 yield_count = __kmp_yield_off_count;
918 __kmp_yielding_on = 1;
919 yield_count = __kmp_yield_on_count;
924 __kmp_yielding_on = 1;
927 TCW_4( __kmp_global.g.g_time.dt.t_value,
928 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
933 KA_TRACE( 10, (
"__kmp_launch_monitor: #3 cleanup\n" ) );
935 #ifdef KMP_BLOCK_SIGNALS
936 status = sigfillset( & new_set );
937 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status );
938 status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
939 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
942 KA_TRACE( 10, (
"__kmp_launch_monitor: #4 finished\n" ) );
944 if( __kmp_global.g.g_abort != 0 ) {
950 KA_TRACE( 10, (
"__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
955 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
956 __kmp_terminate_thread( gtid );
960 KA_TRACE( 10, (
"__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
962 if (__kmp_global.g.g_abort > 0)
963 raise( __kmp_global.g.g_abort );
967 KA_TRACE( 10, (
"__kmp_launch_monitor: #7 exit\n" ) );
973 __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size )
976 pthread_attr_t thread_attr;
980 th->th.th_info.ds.ds_gtid = gtid;
982 #if KMP_STATS_ENABLED
984 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
989 th->th.th_stats = __kmp_stats_list.push_back(gtid);
990 if(KMP_UBER_GTID(gtid)) {
991 __kmp_stats_start_time = tsc_tick_count::now();
992 __kmp_stats_thread_ptr = th->th.th_stats;
997 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
999 #endif // KMP_STATS_ENABLED
1001 if ( KMP_UBER_GTID(gtid) ) {
1002 KA_TRACE( 10, (
"__kmp_create_worker: uber thread (%d)\n", gtid ) );
1003 th -> th.th_info.ds.ds_thread = pthread_self();
1004 __kmp_set_stack_info( gtid, th );
1005 __kmp_check_stack_overlap( th );
1009 KA_TRACE( 10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1013 #ifdef KMP_THREAD_ATTR
1015 status = pthread_attr_init( &thread_attr );
1016 if ( status != 0 ) {
1019 KMP_MSG( CantInitThreadAttrs ),
1024 status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1025 if ( status != 0 ) {
1028 KMP_MSG( CantSetWorkerState ),
1039 stack_size += gtid * __kmp_stkoffset * 2;
1041 KA_TRACE( 10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1042 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1043 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1045 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1046 status = pthread_attr_setstacksize( & thread_attr, stack_size );
1047 # ifdef KMP_BACKUP_STKSIZE
1048 if ( status != 0 ) {
1049 if ( ! __kmp_env_stksize ) {
1050 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1051 __kmp_stksize = KMP_BACKUP_STKSIZE;
1052 KA_TRACE( 10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1053 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1055 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1057 status = pthread_attr_setstacksize( &thread_attr, stack_size );
1061 if ( status != 0 ) {
1064 KMP_MSG( CantSetWorkerStackSize, stack_size ),
1066 KMP_HNT( ChangeWorkerStackSize ),
1075 status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (
void *) th );
1076 if ( status != 0 || ! handle ) {
1077 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1078 if ( status == EINVAL ) {
1081 KMP_MSG( CantSetWorkerStackSize, stack_size ),
1083 KMP_HNT( IncreaseWorkerStackSize ),
1087 if ( status == ENOMEM ) {
1090 KMP_MSG( CantSetWorkerStackSize, stack_size ),
1092 KMP_HNT( DecreaseWorkerStackSize ),
1097 if ( status == EAGAIN ) {
1100 KMP_MSG( NoResourcesForWorkerThread ),
1102 KMP_HNT( Decrease_NUM_THREADS ),
1106 KMP_SYSFAIL(
"pthread_create", status );
1109 th->th.th_info.ds.ds_thread = handle;
1112 #ifdef KMP_THREAD_ATTR
1114 status = pthread_attr_destroy( & thread_attr );
1118 KMP_MSG( CantDestroyThreadAttrs ),
1128 KA_TRACE( 10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1134 __kmp_create_monitor( kmp_info_t *th )
1137 pthread_attr_t thread_attr;
1140 int caller_gtid = __kmp_get_gtid();
1141 int auto_adj_size = FALSE;
1143 KA_TRACE( 10, (
"__kmp_create_monitor: try to create monitor\n" ) );
1147 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1148 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1149 #if KMP_REAL_TIME_FIX
1150 TCW_4( __kmp_global.g.g_time.dt.t_value, -1 );
1152 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1153 #endif // KMP_REAL_TIME_FIX
1155 #ifdef KMP_THREAD_ATTR
1156 if ( __kmp_monitor_stksize == 0 ) {
1157 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1158 auto_adj_size = TRUE;
1160 status = pthread_attr_init( &thread_attr );
1161 if ( status != 0 ) {
1164 KMP_MSG( CantInitThreadAttrs ),
1169 status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1170 if ( status != 0 ) {
1173 KMP_MSG( CantSetMonitorState ),
1179 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1180 status = pthread_attr_getstacksize( & thread_attr, & size );
1181 KMP_CHECK_SYSFAIL(
"pthread_attr_getstacksize", status );
1183 size = __kmp_sys_min_stksize;
1187 if ( __kmp_monitor_stksize == 0 ) {
1188 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1190 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1191 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1194 KA_TRACE( 10, (
"__kmp_create_monitor: default stacksize = %lu bytes,"
1195 "requested stacksize = %lu bytes\n",
1196 size, __kmp_monitor_stksize ) );
1202 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1203 KA_TRACE( 10, (
"__kmp_create_monitor: setting stacksize = %lu bytes,",
1204 __kmp_monitor_stksize ) );
1205 status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1206 if ( status != 0 ) {
1207 if ( auto_adj_size ) {
1208 __kmp_monitor_stksize *= 2;
1213 KMP_MSG( CantSetMonitorStackSize, (
long int) __kmp_monitor_stksize ),
1215 KMP_HNT( ChangeMonitorStackSize ),
1221 status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (
void *) th );
1223 if ( status != 0 ) {
1224 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1225 if ( status == EINVAL ) {
1226 if ( auto_adj_size && ( __kmp_monitor_stksize < (
size_t)0x40000000 ) ) {
1227 __kmp_monitor_stksize *= 2;
1232 KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1234 KMP_HNT( IncreaseMonitorStackSize ),
1238 if ( status == ENOMEM ) {
1241 KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1243 KMP_HNT( DecreaseMonitorStackSize ),
1248 if ( status == EAGAIN ) {
1251 KMP_MSG( NoResourcesForMonitorThread ),
1253 KMP_HNT( DecreaseNumberOfThreadsInUse ),
1257 KMP_SYSFAIL(
"pthread_create", status );
1260 th->th.th_info.ds.ds_thread = handle;
1262 #if KMP_REAL_TIME_FIX
1264 KMP_DEBUG_ASSERT(
sizeof( kmp_uint32 ) ==
sizeof( __kmp_global.g.g_time.dt.t_value ) );
1266 (kmp_uint32
volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1268 #endif // KMP_REAL_TIME_FIX
1270 #ifdef KMP_THREAD_ATTR
1271 status = pthread_attr_destroy( & thread_attr );
1272 if ( status != 0 ) {
1275 KMP_MSG( CantDestroyThreadAttrs ),
1284 KA_TRACE( 10, (
"__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1292 pthread_exit( (
void *)(intptr_t) exit_status );
1295 void __kmp_resume_monitor();
1298 __kmp_reap_monitor( kmp_info_t *th )
1303 KA_TRACE( 10, (
"__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1304 th->th.th_info.ds.ds_thread ) );
1309 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1310 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1321 status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1322 if (status == ESRCH) {
1324 KA_TRACE( 10, (
"__kmp_reap_monitor: monitor does not exist, returning\n") );
1328 __kmp_resume_monitor();
1329 status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1330 if (exit_val != th) {
1333 KMP_MSG( ReapMonitorError ),
1340 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1341 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1343 KA_TRACE( 10, (
"__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1344 th->th.th_info.ds.ds_thread ) );
1351 __kmp_reap_worker( kmp_info_t *th )
1358 KA_TRACE( 10, (
"__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1365 status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1366 if (status == ESRCH) {
1367 KA_TRACE( 10, (
"__kmp_reap_worker: worker T#%d does not exist, returning\n",
1368 th->th.th_info.ds.ds_gtid ) );
1371 KA_TRACE( 10, (
"__kmp_reap_worker: try to join with worker T#%d\n",
1372 th->th.th_info.ds.ds_gtid ) );
1374 status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1377 if ( status != 0 ) {
1380 KMP_MSG( ReapWorkerError ),
1385 if ( exit_val != th ) {
1386 KA_TRACE( 10, (
"__kmp_reap_worker: worker T#%d did not reap properly, "
1388 th->th.th_info.ds.ds_gtid, exit_val ) );
1394 KA_TRACE( 10, (
"__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1403 #if KMP_HANDLE_SIGNALS
1407 __kmp_null_handler(
int signo )
1414 __kmp_team_handler(
int signo )
1416 if ( __kmp_global.g.g_abort == 0 ) {
1419 __kmp_debug_printf(
"__kmp_team_handler: caught signal = %d\n", signo );
1434 if ( __kmp_debug_buf ) {
1435 __kmp_dump_debug_buffer( );
1438 TCW_4( __kmp_global.g.g_abort, signo );
1440 TCW_4( __kmp_global.g.g_done, TRUE );
1445 __kmp_debug_printf(
"__kmp_team_handler: unknown signal type" );
1454 void __kmp_sigaction(
int signum,
const struct sigaction * act,
struct sigaction * oldact ) {
1455 int rc = sigaction( signum, act, oldact );
1456 KMP_CHECK_SYSFAIL_ERRNO(
"sigaction", rc );
1461 __kmp_install_one_handler(
int sig, sig_func_t handler_func,
int parallel_init )
1464 KB_TRACE( 60, (
"__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1465 if ( parallel_init ) {
1466 struct sigaction new_action;
1467 struct sigaction old_action;
1468 new_action.sa_handler = handler_func;
1469 new_action.sa_flags = 0;
1470 sigfillset( & new_action.sa_mask );
1471 __kmp_sigaction( sig, & new_action, & old_action );
1472 if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1473 sigaddset( & __kmp_sigset, sig );
1476 __kmp_sigaction( sig, & old_action, NULL );
1480 __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1487 __kmp_remove_one_handler(
int sig )
1489 KB_TRACE( 60, (
"__kmp_remove_one_handler( %d )\n", sig ) );
1490 if ( sigismember( & __kmp_sigset, sig ) ) {
1491 struct sigaction old;
1493 __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1494 if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1496 KB_TRACE( 10, (
"__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1497 __kmp_sigaction( sig, & old, NULL );
1499 sigdelset( & __kmp_sigset, sig );
1506 __kmp_install_signals(
int parallel_init )
1508 KB_TRACE( 10, (
"__kmp_install_signals( %d )\n", parallel_init ) );
1509 if ( __kmp_handle_signals || ! parallel_init ) {
1512 sigemptyset( & __kmp_sigset );
1513 __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1514 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1515 __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1516 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1517 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1518 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1519 __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1520 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1522 __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1524 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1526 __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1533 __kmp_remove_signals(
void )
1536 KB_TRACE( 10, (
"__kmp_remove_signals()\n" ) );
1537 for ( sig = 1; sig < NSIG; ++ sig ) {
1538 __kmp_remove_one_handler( sig );
1543 #endif // KMP_HANDLE_SIGNALS
1549 __kmp_enable(
int new_state )
1551 #ifdef KMP_CANCEL_THREADS
1552 int status, old_state;
1553 status = pthread_setcancelstate( new_state, & old_state );
1554 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
1555 KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1560 __kmp_disable(
int * old_state )
1562 #ifdef KMP_CANCEL_THREADS
1564 status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1565 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
1573 __kmp_atfork_prepare (
void)
1579 __kmp_atfork_parent (
void)
1590 __kmp_atfork_child (
void)
1598 __kmp_init_runtime = FALSE;
1599 __kmp_init_monitor = 0;
1600 __kmp_init_parallel = FALSE;
1601 __kmp_init_middle = FALSE;
1602 __kmp_init_serial = FALSE;
1603 TCW_4(__kmp_init_gtid, FALSE);
1604 __kmp_init_common = FALSE;
1606 TCW_4(__kmp_init_user_locks, FALSE);
1607 #if ! KMP_USE_DYNAMIC_LOCK
1608 __kmp_user_lock_table.used = 1;
1609 __kmp_user_lock_table.allocated = 0;
1610 __kmp_user_lock_table.table = NULL;
1611 __kmp_lock_blocks = NULL;
1615 TCW_4(__kmp_nth, 0);
1619 KA_TRACE( 10, (
"__kmp_atfork_child: checking cache address list %p\n",
1620 __kmp_threadpriv_cache_list ) );
1622 while ( __kmp_threadpriv_cache_list != NULL ) {
1624 if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1625 KC_TRACE( 50, (
"__kmp_atfork_child: zeroing cache at address %p\n",
1626 &(*__kmp_threadpriv_cache_list -> addr) ) );
1628 *__kmp_threadpriv_cache_list -> addr = NULL;
1630 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1633 __kmp_init_runtime = FALSE;
1636 __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1637 __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1638 __kmp_init_bootstrap_lock( &__kmp_console_lock );
1653 __kmp_register_atfork(
void) {
1654 if ( __kmp_need_register_atfork ) {
1655 int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1656 KMP_CHECK_SYSFAIL(
"pthread_atfork", status );
1657 __kmp_need_register_atfork = FALSE;
1662 __kmp_suspend_initialize(
void )
1665 status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1666 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status );
1667 status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1668 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status );
1672 __kmp_suspend_initialize_thread( kmp_info_t *th )
1674 if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1678 status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1679 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status );
1680 status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1681 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status );
1682 *(
volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1687 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1689 if(th->th.th_suspend_init_count > __kmp_fork_count) {
1694 status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1695 if ( status != 0 && status != EBUSY ) {
1696 KMP_SYSFAIL(
"pthread_cond_destroy", status );
1698 status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1699 if ( status != 0 && status != EBUSY ) {
1700 KMP_SYSFAIL(
"pthread_mutex_destroy", status );
1702 --th->th.th_suspend_init_count;
1703 KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1711 static inline void __kmp_suspend_template(
int th_gtid, C *flag )
1714 kmp_info_t *th = __kmp_threads[th_gtid];
1716 typename C::flag_t old_spin;
1718 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1720 __kmp_suspend_initialize_thread( th );
1722 status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1723 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1725 KF_TRACE( 10, (
"__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1726 th_gtid, flag->get() ) );
1731 old_spin = flag->set_sleeping();
1733 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x, was %x\n",
1734 th_gtid, flag->get(), *(flag->get()), old_spin ) );
1736 if ( flag->done_check_val(old_spin) ) {
1737 old_spin = flag->unset_sleeping();
1738 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1739 th_gtid, flag->get()) );
1745 int deactivated = FALSE;
1746 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
1747 while ( flag->is_sleeping() ) {
1748 #ifdef DEBUG_SUSPEND
1750 __kmp_suspend_count++;
1751 __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1752 __kmp_printf(
"__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1755 if ( ! deactivated ) {
1756 th->th.th_active = FALSE;
1757 if ( th->th.th_active_in_pool ) {
1758 th->th.th_active_in_pool = FALSE;
1759 KMP_TEST_THEN_DEC32(
1760 (kmp_int32 *) &__kmp_thread_pool_active_nth );
1761 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1768 #if USE_SUSPEND_TIMEOUT
1769 struct timespec now;
1770 struct timeval tval;
1773 status = gettimeofday( &tval, NULL );
1774 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
1775 TIMEVAL_TO_TIMESPEC( &tval, &now );
1777 msecs = (4*__kmp_dflt_blocktime) + 200;
1778 now.tv_sec += msecs / 1000;
1779 now.tv_nsec += (msecs % 1000)*1000;
1781 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1783 status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1785 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1787 status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1790 if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1791 KMP_SYSFAIL(
"pthread_cond_wait", status );
1794 if (status == ETIMEDOUT) {
1795 if ( flag->is_sleeping() ) {
1796 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1798 KF_TRACE( 2, (
"__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1801 }
else if ( flag->is_sleeping() ) {
1802 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1808 if ( deactivated ) {
1809 th->th.th_active = TRUE;
1810 if ( TCR_4(th->th.th_in_pool) ) {
1811 KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1812 th->th.th_active_in_pool = TRUE;
1817 #ifdef DEBUG_SUSPEND
1820 __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1821 __kmp_printf(
"__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1826 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1827 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1829 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1832 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
1833 __kmp_suspend_template(th_gtid, flag);
1835 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
1836 __kmp_suspend_template(th_gtid, flag);
1838 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
1839 __kmp_suspend_template(th_gtid, flag);
1847 static inline void __kmp_resume_template(
int target_gtid, C *flag )
1849 kmp_info_t *th = __kmp_threads[target_gtid];
1853 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1856 KF_TRACE( 30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1857 KMP_DEBUG_ASSERT( gtid != target_gtid );
1859 __kmp_suspend_initialize_thread( th );
1861 status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1862 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1865 flag = (C *)th->th.th_sleep_loc;
1869 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1870 gtid, target_gtid, NULL ) );
1871 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1872 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1876 typename C::flag_t old_spin = flag->unset_sleeping();
1877 if ( ! flag->is_sleeping_val(old_spin) ) {
1878 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1880 gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1882 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1883 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1886 KF_TRACE( 5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1888 gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1890 TCW_PTR(th->th.th_sleep_loc, NULL);
1893 #ifdef DEBUG_SUSPEND
1896 __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1897 __kmp_printf(
"__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1902 status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1903 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status );
1904 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1905 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1906 KF_TRACE( 30, (
"__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1907 gtid, target_gtid ) );
1910 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
1911 __kmp_resume_template(target_gtid, flag);
1913 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
1914 __kmp_resume_template(target_gtid, flag);
1916 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
1917 __kmp_resume_template(target_gtid, flag);
1921 __kmp_resume_monitor()
1926 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1927 KF_TRACE( 30, (
"__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1928 gtid, KMP_GTID_MONITOR ) );
1929 KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1931 status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1932 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1933 #ifdef DEBUG_SUSPEND
1936 __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1937 __kmp_printf(
"__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1940 status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1941 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status );
1942 status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1943 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1944 KF_TRACE( 30, (
"__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1945 gtid, KMP_GTID_MONITOR ) );
1952 __kmp_yield(
int cond )
1954 if (cond && __kmp_yielding_on) {
1963 __kmp_gtid_set_specific(
int gtid )
1966 KMP_ASSERT( __kmp_init_runtime );
1967 status = pthread_setspecific( __kmp_gtid_threadprivate_key, (
void*)(intptr_t)(gtid+1) );
1968 KMP_CHECK_SYSFAIL(
"pthread_setspecific", status );
1972 __kmp_gtid_get_specific()
1975 if ( !__kmp_init_runtime ) {
1976 KA_TRACE( 50, (
"__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1977 return KMP_GTID_SHUTDOWN;
1979 gtid = (int)(
size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1981 gtid = KMP_GTID_DNE;
1986 KA_TRACE( 50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
1987 __kmp_gtid_threadprivate_key, gtid ));
1995 __kmp_read_cpu_time(
void )
2002 return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
2006 __kmp_read_system_info(
struct kmp_sys_info *info )
2009 struct rusage r_usage;
2011 memset( info, 0,
sizeof( *info ) );
2013 status = getrusage( RUSAGE_SELF, &r_usage);
2014 KMP_CHECK_SYSFAIL_ERRNO(
"getrusage", status );
2016 info->maxrss = r_usage.ru_maxrss;
2017 info->minflt = r_usage.ru_minflt;
2018 info->majflt = r_usage.ru_majflt;
2019 info->nswap = r_usage.ru_nswap;
2020 info->inblock = r_usage.ru_inblock;
2021 info->oublock = r_usage.ru_oublock;
2022 info->nvcsw = r_usage.ru_nvcsw;
2023 info->nivcsw = r_usage.ru_nivcsw;
2025 return (status != 0);
2033 __kmp_read_system_time(
double *delta )
2036 struct timeval tval;
2037 struct timespec stop;
2040 status = gettimeofday( &tval, NULL );
2041 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2042 TIMEVAL_TO_TIMESPEC( &tval, &stop );
2043 t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
2044 *delta = (t_ns * 1e-9);
2048 __kmp_clear_system_time(
void )
2050 struct timeval tval;
2052 status = gettimeofday( &tval, NULL );
2053 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2054 TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2063 __kmp_tv_threadprivate_store( kmp_info_t *th,
void *global_addr,
void *thread_addr )
2067 p = (
struct tv_data *) __kmp_allocate(
sizeof( *p ) );
2069 p->u.tp.global_addr = global_addr;
2070 p->u.tp.thread_addr = thread_addr;
2072 p->type = (
void *) 1;
2074 p->next = th->th.th_local.tv_data;
2075 th->th.th_local.tv_data = p;
2077 if ( p->next == 0 ) {
2078 int rc = pthread_setspecific( __kmp_tv_key, p );
2079 KMP_CHECK_SYSFAIL(
"pthread_setspecific", rc );
2089 __kmp_get_xproc(
void ) {
2095 r = sysconf( _SC_NPROCESSORS_ONLN );
2103 host_basic_info_data_t info;
2104 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2105 rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2106 if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2109 r = info.avail_cpus;
2111 KMP_WARNING( CantGetNumAvailCPU );
2112 KMP_INFORM( AssumedNumCPU );
2115 #elif KMP_OS_FREEBSD
2117 int mib[] = { CTL_HW, HW_NCPU };
2118 size_t len =
sizeof( r );
2119 if ( sysctl( mib, 2, &r, &len, NULL, 0 ) < 0 ) {
2121 KMP_WARNING( CantGetNumAvailCPU );
2122 KMP_INFORM( AssumedNumCPU );
2127 #error "Unknown or unsupported OS."
2131 return r > 0 ? r : 2;
2136 __kmp_read_from_file(
char const *path,
char const *format, ... )
2141 va_start(args, format);
2142 FILE *f = fopen(path,
"rb");
2145 result = vfscanf(f, format, args);
2152 __kmp_runtime_initialize(
void )
2155 pthread_mutexattr_t mutex_attr;
2156 pthread_condattr_t cond_attr;
2158 if ( __kmp_init_runtime ) {
2162 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2163 if ( ! __kmp_cpuinfo.initialized ) {
2164 __kmp_query_cpuid( &__kmp_cpuinfo );
2168 __kmp_xproc = __kmp_get_xproc();
2170 if ( sysconf( _SC_THREADS ) ) {
2173 __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2174 if ( __kmp_sys_max_nth == -1 ) {
2176 __kmp_sys_max_nth = INT_MAX;
2178 else if ( __kmp_sys_max_nth <= 1 ) {
2180 __kmp_sys_max_nth = KMP_MAX_NTH;
2184 __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2185 if ( __kmp_sys_min_stksize <= 1 ) {
2186 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2191 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2196 int rc = pthread_key_create( & __kmp_tv_key, 0 );
2197 KMP_CHECK_SYSFAIL(
"pthread_key_create", rc );
2201 status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2202 KMP_CHECK_SYSFAIL(
"pthread_key_create", status );
2203 status = pthread_mutexattr_init( & mutex_attr );
2204 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status );
2205 status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2206 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status );
2207 status = pthread_condattr_init( & cond_attr );
2208 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status );
2209 status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2210 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status );
2212 __kmp_itt_initialize();
2215 __kmp_init_runtime = TRUE;
2219 __kmp_runtime_destroy(
void )
2223 if ( ! __kmp_init_runtime ) {
2228 __kmp_itt_destroy();
2231 status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2232 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status );
2234 status = pthread_key_delete( __kmp_tv_key );
2235 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status );
2238 status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2239 if ( status != 0 && status != EBUSY ) {
2240 KMP_SYSFAIL(
"pthread_mutex_destroy", status );
2242 status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2243 if ( status != 0 && status != EBUSY ) {
2244 KMP_SYSFAIL(
"pthread_cond_destroy", status );
2246 #if KMP_AFFINITY_SUPPORTED
2247 __kmp_affinity_uninitialize();
2250 __kmp_init_runtime = FALSE;
2257 __kmp_thread_sleep(
int millis )
2259 sleep( ( millis + 500 ) / 1000 );
2264 __kmp_elapsed(
double *t )
2267 # ifdef FIX_SGI_CLOCK
2270 status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2271 KMP_CHECK_SYSFAIL_ERRNO(
"clock_gettime", status );
2272 *t = (double) ts.tv_nsec * (1.0 / (
double) KMP_NSEC_PER_SEC) +
2277 status = gettimeofday( & tv, NULL );
2278 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2279 *t = (double) tv.tv_usec * (1.0 / (
double) KMP_USEC_PER_SEC) +
2286 __kmp_elapsed_tick(
double *t )
2288 *t = 1 / (double) CLOCKS_PER_SEC;
2296 __kmp_is_address_mapped(
void * addr ) {
2308 char * name = __kmp_str_format(
"/proc/%d/maps", getpid() );
2311 file = fopen( name,
"r" );
2312 KMP_ASSERT( file != NULL );
2316 void * beginning = NULL;
2317 void * ending = NULL;
2320 rc = fscanf( file,
"%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2324 KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 );
2327 if ( ( addr >= beginning ) && ( addr < ending ) ) {
2329 if ( strcmp( perms,
"rw" ) == 0 ) {
2340 KMP_INTERNAL_FREE( name );
2354 (vm_address_t)( addr ),
2356 (vm_address_t)( & buffer ),
2364 #elif KMP_OS_FREEBSD
2371 #error "Unknown or unsupported OS"
2379 #ifdef USE_LOAD_BALANCE
2390 __kmp_get_load_balance(
int max )
2395 int res = getloadavg( averages, 3 );
2400 if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2401 ret_avg = averages[0];
2402 }
else if ( ( __kmp_load_balance_interval >= 180
2403 && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2404 ret_avg = averages[1];
2405 }
else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2406 ret_avg = averages[2];
2420 __kmp_get_load_balance(
int max )
2422 static int permanent_error = 0;
2424 static int glb_running_threads = 0;
2425 static double glb_call_time = 0;
2427 int running_threads = 0;
2429 DIR * proc_dir = NULL;
2430 struct dirent * proc_entry = NULL;
2432 kmp_str_buf_t task_path;
2433 DIR * task_dir = NULL;
2434 struct dirent * task_entry = NULL;
2435 int task_path_fixed_len;
2437 kmp_str_buf_t stat_path;
2439 int stat_path_fixed_len;
2441 int total_processes = 0;
2442 int total_threads = 0;
2444 double call_time = 0.0;
2446 __kmp_str_buf_init( & task_path );
2447 __kmp_str_buf_init( & stat_path );
2449 __kmp_elapsed( & call_time );
2451 if ( glb_call_time &&
2452 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2453 running_threads = glb_running_threads;
2457 glb_call_time = call_time;
2460 if ( permanent_error ) {
2461 running_threads = -1;
2470 proc_dir = opendir(
"/proc" );
2471 if ( proc_dir == NULL ) {
2474 running_threads = -1;
2475 permanent_error = 1;
2480 __kmp_str_buf_cat( & task_path,
"/proc/", 6 );
2481 task_path_fixed_len = task_path.used;
2483 proc_entry = readdir( proc_dir );
2484 while ( proc_entry != NULL ) {
2487 if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2495 KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name,
"1" ) == 0 );
2498 task_path.used = task_path_fixed_len;
2499 __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2500 __kmp_str_buf_cat( & task_path,
"/task", 5 );
2502 task_dir = opendir( task_path.str );
2503 if ( task_dir == NULL ) {
2511 if ( strcmp( proc_entry->d_name,
"1" ) == 0 ) {
2512 running_threads = -1;
2513 permanent_error = 1;
2518 __kmp_str_buf_clear( & stat_path );
2519 __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2520 __kmp_str_buf_cat( & stat_path,
"/", 1 );
2521 stat_path_fixed_len = stat_path.used;
2523 task_entry = readdir( task_dir );
2524 while ( task_entry != NULL ) {
2526 if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2533 stat_path.used = stat_path_fixed_len;
2534 __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2535 __kmp_str_buf_cat( & stat_path,
"/stat", 5 );
2539 stat_file = open( stat_path.str, O_RDONLY );
2540 if ( stat_file == -1 ) {
2573 len = read( stat_file, buffer,
sizeof( buffer ) - 1 );
2580 char * close_parent = strstr( buffer,
") " );
2581 if ( close_parent != NULL ) {
2582 char state = * ( close_parent + 2 );
2583 if ( state ==
'R' ) {
2585 if ( running_threads >= max ) {
2595 task_entry = readdir( task_dir );
2597 closedir( task_dir );
2601 proc_entry = readdir( proc_dir );
2609 KMP_DEBUG_ASSERT( running_threads > 0 );
2610 if ( running_threads <= 0 ) {
2611 running_threads = 1;
2615 if ( proc_dir != NULL ) {
2616 closedir( proc_dir );
2618 __kmp_str_buf_free( & task_path );
2619 if ( task_dir != NULL ) {
2620 closedir( task_dir );
2622 __kmp_str_buf_free( & stat_path );
2623 if ( stat_file != -1 ) {
2627 glb_running_threads = running_threads;
2629 return running_threads;
2633 # endif // KMP_OS_DARWIN
2635 #endif // USE_LOAD_BALANCE
2638 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
2640 int __kmp_invoke_microtask( microtask_t pkfn,
int gtid,
int tid,
int argc,
2643 int argc_full = argc + 2;
2646 ffi_type *types[argc_full];
2647 void *args[argc_full];
2651 for (i = 0; i < argc_full; i++)
2652 types[i] = &ffi_type_pointer;
2660 for (i = 0; i < argc; i++)
2661 args[2 + i] = &p_argv[i];
2663 if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, argc_full,
2664 &ffi_type_void, types) != FFI_OK)
2667 ffi_call(&cif, (
void (*)(
void))pkfn, NULL, args);
2672 #endif // KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
2674 #if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
2679 __kmp_invoke_microtask( microtask_t pkfn,
2681 int argc,
void *p_argv[] ) {
2684 fprintf(stderr,
"Too many args to microtask: %d!\n", argc);
2688 (*pkfn)(>id, &tid);
2691 (*pkfn)(>id, &tid, p_argv[0]);
2694 (*pkfn)(>id, &tid, p_argv[0], p_argv[1]);
2697 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]);
2700 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2703 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2706 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2710 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2711 p_argv[5], p_argv[6]);
2714 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2715 p_argv[5], p_argv[6], p_argv[7]);
2718 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2719 p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2722 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2723 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2726 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2727 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2730 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2731 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2735 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2736 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2737 p_argv[11], p_argv[12]);
2740 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2741 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2742 p_argv[11], p_argv[12], p_argv[13]);
2745 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2746 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2747 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.