43 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
45 # include <sys/syscall.h>
63 __kmp_validate_locks(
void )
69 x = ~((kmp_uint32) 0) - 2;
72 for (i = 0; i < 8; ++i, ++x, ++y) {
73 kmp_uint32 z = (x - y);
77 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
95 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
97 return DYNA_LOCK_STRIP(TCR_4( lck->lk.poll )) - 1;
101 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
103 return lck->lk.depth_locked != -1;
106 __forceinline
static void
107 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
111 #ifdef USE_LOCK_PROFILE
112 kmp_uint32 curr = TCR_4( lck->lk.poll );
113 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
114 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
118 if ( ( lck->lk.poll == DYNA_LOCK_FREE(tas) )
119 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) {
120 KMP_FSYNC_ACQUIRED(lck);
125 KMP_FSYNC_PREPARE( lck );
126 KMP_INIT_YIELD( spins );
127 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
132 KMP_YIELD_SPIN( spins );
135 while ( ( lck->lk.poll != DYNA_LOCK_FREE(tas) ) ||
136 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) ) {
140 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
145 KMP_YIELD_SPIN( spins );
148 KMP_FSYNC_ACQUIRED( lck );
152 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
154 __kmp_acquire_tas_lock_timed_template( lck, gtid );
158 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
160 char const *
const func =
"omp_set_lock";
161 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
162 && __kmp_is_tas_lock_nestable( lck ) ) {
163 KMP_FATAL( LockNestableUsedAsSimple, func );
165 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
166 KMP_FATAL( LockIsAlreadyOwned, func );
168 __kmp_acquire_tas_lock( lck, gtid );
172 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
174 if ( ( lck->lk.poll == DYNA_LOCK_FREE(tas) )
175 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) {
176 KMP_FSYNC_ACQUIRED( lck );
183 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
185 char const *
const func =
"omp_test_lock";
186 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
187 && __kmp_is_tas_lock_nestable( lck ) ) {
188 KMP_FATAL( LockNestableUsedAsSimple, func );
190 return __kmp_test_tas_lock( lck, gtid );
194 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
198 KMP_FSYNC_RELEASING(lck);
199 KMP_ST_REL32( &(lck->lk.poll), DYNA_LOCK_FREE(tas) );
202 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
204 return KMP_LOCK_RELEASED;
208 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
210 char const *
const func =
"omp_unset_lock";
212 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
213 && __kmp_is_tas_lock_nestable( lck ) ) {
214 KMP_FATAL( LockNestableUsedAsSimple, func );
216 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
217 KMP_FATAL( LockUnsettingFree, func );
219 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
220 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
221 KMP_FATAL( LockUnsettingSetByAnother, func );
223 return __kmp_release_tas_lock( lck, gtid );
227 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
229 TCW_4( lck->lk.poll, DYNA_LOCK_FREE(tas) );
233 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
235 __kmp_init_tas_lock( lck );
239 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
245 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
247 char const *
const func =
"omp_destroy_lock";
248 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
249 && __kmp_is_tas_lock_nestable( lck ) ) {
250 KMP_FATAL( LockNestableUsedAsSimple, func );
252 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
253 KMP_FATAL( LockStillOwned, func );
255 __kmp_destroy_tas_lock( lck );
264 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
266 KMP_DEBUG_ASSERT( gtid >= 0 );
268 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
269 lck->lk.depth_locked += 1;
272 __kmp_acquire_tas_lock_timed_template( lck, gtid );
273 lck->lk.depth_locked = 1;
278 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
280 char const *
const func =
"omp_set_nest_lock";
281 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
282 KMP_FATAL( LockSimpleUsedAsNestable, func );
284 __kmp_acquire_nested_tas_lock( lck, gtid );
288 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
292 KMP_DEBUG_ASSERT( gtid >= 0 );
294 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
295 retval = ++lck->lk.depth_locked;
297 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
302 retval = lck->lk.depth_locked = 1;
308 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
310 char const *
const func =
"omp_test_nest_lock";
311 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
312 KMP_FATAL( LockSimpleUsedAsNestable, func );
314 return __kmp_test_nested_tas_lock( lck, gtid );
318 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
320 KMP_DEBUG_ASSERT( gtid >= 0 );
323 if ( --(lck->lk.depth_locked) == 0 ) {
324 __kmp_release_tas_lock( lck, gtid );
325 return KMP_LOCK_RELEASED;
327 return KMP_LOCK_STILL_HELD;
331 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
333 char const *
const func =
"omp_unset_nest_lock";
335 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
336 KMP_FATAL( LockSimpleUsedAsNestable, func );
338 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
339 KMP_FATAL( LockUnsettingFree, func );
341 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
342 KMP_FATAL( LockUnsettingSetByAnother, func );
344 return __kmp_release_nested_tas_lock( lck, gtid );
348 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
350 __kmp_init_tas_lock( lck );
351 lck->lk.depth_locked = 0;
355 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
357 __kmp_init_nested_tas_lock( lck );
361 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
363 __kmp_destroy_tas_lock( lck );
364 lck->lk.depth_locked = 0;
368 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
370 char const *
const func =
"omp_destroy_nest_lock";
371 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
372 KMP_FATAL( LockSimpleUsedAsNestable, func );
374 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
375 KMP_FATAL( LockStillOwned, func );
377 __kmp_destroy_nested_tas_lock( lck );
381 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
392 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
394 return DYNA_LOCK_STRIP(( TCR_4( lck->lk.poll ) >> 1 )) - 1;
398 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
400 return lck->lk.depth_locked != -1;
403 __forceinline
static void
404 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
406 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
410 #ifdef USE_LOCK_PROFILE
411 kmp_uint32 curr = TCR_4( lck->lk.poll );
412 if ( ( curr != 0 ) && ( curr != gtid_code ) )
413 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
417 KMP_FSYNC_PREPARE( lck );
418 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
419 lck, lck->lk.poll, gtid ) );
423 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex),
424 DYNA_LOCK_BUSY(gtid_code, futex) ) ) != DYNA_LOCK_FREE(futex) ) {
426 kmp_int32 cond = DYNA_LOCK_STRIP(poll_val) & 1;
427 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
428 lck, gtid, poll_val, cond ) );
443 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ), poll_val, poll_val | DYNA_LOCK_BUSY(1, futex) ) ) {
444 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
445 lck, lck->lk.poll, gtid ) );
448 poll_val |= DYNA_LOCK_BUSY(1, futex);
450 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
451 lck, lck->lk.poll, gtid ) );
454 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
455 lck, gtid, poll_val ) );
458 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
459 poll_val, NULL, NULL, 0 ) ) != 0 ) {
460 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
461 lck, gtid, poll_val, rc, errno ) );
465 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
466 lck, gtid, poll_val ) );
476 KMP_FSYNC_ACQUIRED( lck );
477 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
478 lck, lck->lk.poll, gtid ) );
482 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
484 __kmp_acquire_futex_lock_timed_template( lck, gtid );
488 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
490 char const *
const func =
"omp_set_lock";
491 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
492 && __kmp_is_futex_lock_nestable( lck ) ) {
493 KMP_FATAL( LockNestableUsedAsSimple, func );
495 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
496 KMP_FATAL( LockIsAlreadyOwned, func );
498 __kmp_acquire_futex_lock( lck, gtid );
502 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
504 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex), DYNA_LOCK_BUSY(gtid+1, futex) << 1 ) ) {
505 KMP_FSYNC_ACQUIRED( lck );
512 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
514 char const *
const func =
"omp_test_lock";
515 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
516 && __kmp_is_futex_lock_nestable( lck ) ) {
517 KMP_FATAL( LockNestableUsedAsSimple, func );
519 return __kmp_test_futex_lock( lck, gtid );
523 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
527 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
528 lck, lck->lk.poll, gtid ) );
530 KMP_FSYNC_RELEASING(lck);
532 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex) );
534 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
535 lck, gtid, poll_val ) );
537 if ( DYNA_LOCK_STRIP(poll_val) & 1 ) {
538 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
540 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, DYNA_LOCK_BUSY(1, futex), NULL, NULL, 0 );
545 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
546 lck, lck->lk.poll, gtid ) );
548 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
550 return KMP_LOCK_RELEASED;
554 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
556 char const *
const func =
"omp_unset_lock";
558 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
559 && __kmp_is_futex_lock_nestable( lck ) ) {
560 KMP_FATAL( LockNestableUsedAsSimple, func );
562 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
563 KMP_FATAL( LockUnsettingFree, func );
565 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
566 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
567 KMP_FATAL( LockUnsettingSetByAnother, func );
569 return __kmp_release_futex_lock( lck, gtid );
573 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
575 TCW_4( lck->lk.poll, DYNA_LOCK_FREE(futex) );
579 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
581 __kmp_init_futex_lock( lck );
585 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
591 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
593 char const *
const func =
"omp_destroy_lock";
594 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
595 && __kmp_is_futex_lock_nestable( lck ) ) {
596 KMP_FATAL( LockNestableUsedAsSimple, func );
598 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
599 KMP_FATAL( LockStillOwned, func );
601 __kmp_destroy_futex_lock( lck );
610 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
612 KMP_DEBUG_ASSERT( gtid >= 0 );
614 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
615 lck->lk.depth_locked += 1;
618 __kmp_acquire_futex_lock_timed_template( lck, gtid );
619 lck->lk.depth_locked = 1;
624 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
626 char const *
const func =
"omp_set_nest_lock";
627 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
628 KMP_FATAL( LockSimpleUsedAsNestable, func );
630 __kmp_acquire_nested_futex_lock( lck, gtid );
634 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
638 KMP_DEBUG_ASSERT( gtid >= 0 );
640 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
641 retval = ++lck->lk.depth_locked;
643 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
648 retval = lck->lk.depth_locked = 1;
654 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
656 char const *
const func =
"omp_test_nest_lock";
657 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
658 KMP_FATAL( LockSimpleUsedAsNestable, func );
660 return __kmp_test_nested_futex_lock( lck, gtid );
664 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
666 KMP_DEBUG_ASSERT( gtid >= 0 );
669 if ( --(lck->lk.depth_locked) == 0 ) {
670 __kmp_release_futex_lock( lck, gtid );
671 return KMP_LOCK_RELEASED;
673 return KMP_LOCK_STILL_HELD;
677 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
679 char const *
const func =
"omp_unset_nest_lock";
681 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
682 KMP_FATAL( LockSimpleUsedAsNestable, func );
684 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
685 KMP_FATAL( LockUnsettingFree, func );
687 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
688 KMP_FATAL( LockUnsettingSetByAnother, func );
690 return __kmp_release_nested_futex_lock( lck, gtid );
694 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
696 __kmp_init_futex_lock( lck );
697 lck->lk.depth_locked = 0;
701 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
703 __kmp_init_nested_futex_lock( lck );
707 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
709 __kmp_destroy_futex_lock( lck );
710 lck->lk.depth_locked = 0;
714 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
716 char const *
const func =
"omp_destroy_nest_lock";
717 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
718 KMP_FATAL( LockSimpleUsedAsNestable, func );
720 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
721 KMP_FATAL( LockStillOwned, func );
723 __kmp_destroy_nested_futex_lock( lck );
726 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
733 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
735 return TCR_4( lck->lk.owner_id ) - 1;
739 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
741 return lck->lk.depth_locked != -1;
745 __kmp_bakery_check(kmp_uint value, kmp_uint checker)
747 register kmp_uint32 pause;
749 if (value == checker) {
752 for (pause = checker - value; pause != 0; --pause);
756 __forceinline
static void
757 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
759 kmp_uint32 my_ticket;
762 my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
764 #ifdef USE_LOCK_PROFILE
765 if ( TCR_4( lck->lk.now_serving ) != my_ticket )
766 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
770 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
771 KMP_FSYNC_ACQUIRED(lck);
774 KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
775 KMP_FSYNC_ACQUIRED(lck);
779 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
781 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
785 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
787 char const *
const func =
"omp_set_lock";
788 if ( lck->lk.initialized != lck ) {
789 KMP_FATAL( LockIsUninitialized, func );
791 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
792 KMP_FATAL( LockNestableUsedAsSimple, func );
794 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
795 KMP_FATAL( LockIsAlreadyOwned, func );
798 __kmp_acquire_ticket_lock( lck, gtid );
800 lck->lk.owner_id = gtid + 1;
804 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
806 kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
807 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
808 kmp_uint32 next_ticket = my_ticket + 1;
809 if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
810 my_ticket, next_ticket ) ) {
811 KMP_FSYNC_ACQUIRED( lck );
819 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
821 char const *
const func =
"omp_test_lock";
822 if ( lck->lk.initialized != lck ) {
823 KMP_FATAL( LockIsUninitialized, func );
825 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
826 KMP_FATAL( LockNestableUsedAsSimple, func );
829 int retval = __kmp_test_ticket_lock( lck, gtid );
832 lck->lk.owner_id = gtid + 1;
838 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
844 KMP_FSYNC_RELEASING(lck);
845 distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
847 KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
852 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
853 return KMP_LOCK_RELEASED;
857 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
859 char const *
const func =
"omp_unset_lock";
861 if ( lck->lk.initialized != lck ) {
862 KMP_FATAL( LockIsUninitialized, func );
864 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
865 KMP_FATAL( LockNestableUsedAsSimple, func );
867 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
868 KMP_FATAL( LockUnsettingFree, func );
870 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
871 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
872 KMP_FATAL( LockUnsettingSetByAnother, func );
874 lck->lk.owner_id = 0;
875 return __kmp_release_ticket_lock( lck, gtid );
879 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
881 lck->lk.location = NULL;
882 TCW_4( lck->lk.next_ticket, 0 );
883 TCW_4( lck->lk.now_serving, 0 );
884 lck->lk.owner_id = 0;
885 lck->lk.depth_locked = -1;
886 lck->lk.initialized = (kmp_ticket_lock *)lck;
890 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
892 __kmp_init_ticket_lock( lck );
896 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
898 lck->lk.initialized = NULL;
899 lck->lk.location = NULL;
900 lck->lk.next_ticket = 0;
901 lck->lk.now_serving = 0;
902 lck->lk.owner_id = 0;
903 lck->lk.depth_locked = -1;
907 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
909 char const *
const func =
"omp_destroy_lock";
910 if ( lck->lk.initialized != lck ) {
911 KMP_FATAL( LockIsUninitialized, func );
913 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
914 KMP_FATAL( LockNestableUsedAsSimple, func );
916 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
917 KMP_FATAL( LockStillOwned, func );
919 __kmp_destroy_ticket_lock( lck );
928 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
930 KMP_DEBUG_ASSERT( gtid >= 0 );
932 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
933 lck->lk.depth_locked += 1;
936 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
938 lck->lk.depth_locked = 1;
940 lck->lk.owner_id = gtid + 1;
945 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
947 char const *
const func =
"omp_set_nest_lock";
948 if ( lck->lk.initialized != lck ) {
949 KMP_FATAL( LockIsUninitialized, func );
951 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
952 KMP_FATAL( LockSimpleUsedAsNestable, func );
954 __kmp_acquire_nested_ticket_lock( lck, gtid );
958 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
962 KMP_DEBUG_ASSERT( gtid >= 0 );
964 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
965 retval = ++lck->lk.depth_locked;
967 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
972 retval = lck->lk.depth_locked = 1;
974 lck->lk.owner_id = gtid + 1;
980 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
983 char const *
const func =
"omp_test_nest_lock";
984 if ( lck->lk.initialized != lck ) {
985 KMP_FATAL( LockIsUninitialized, func );
987 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
988 KMP_FATAL( LockSimpleUsedAsNestable, func );
990 return __kmp_test_nested_ticket_lock( lck, gtid );
994 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
996 KMP_DEBUG_ASSERT( gtid >= 0 );
999 if ( --(lck->lk.depth_locked) == 0 ) {
1001 lck->lk.owner_id = 0;
1002 __kmp_release_ticket_lock( lck, gtid );
1003 return KMP_LOCK_RELEASED;
1005 return KMP_LOCK_STILL_HELD;
1009 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1011 char const *
const func =
"omp_unset_nest_lock";
1013 if ( lck->lk.initialized != lck ) {
1014 KMP_FATAL( LockIsUninitialized, func );
1016 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1017 KMP_FATAL( LockSimpleUsedAsNestable, func );
1019 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1020 KMP_FATAL( LockUnsettingFree, func );
1022 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1023 KMP_FATAL( LockUnsettingSetByAnother, func );
1025 return __kmp_release_nested_ticket_lock( lck, gtid );
1029 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1031 __kmp_init_ticket_lock( lck );
1032 lck->lk.depth_locked = 0;
1036 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1038 __kmp_init_nested_ticket_lock( lck );
1042 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1044 __kmp_destroy_ticket_lock( lck );
1045 lck->lk.depth_locked = 0;
1049 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1051 char const *
const func =
"omp_destroy_nest_lock";
1052 if ( lck->lk.initialized != lck ) {
1053 KMP_FATAL( LockIsUninitialized, func );
1055 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1056 KMP_FATAL( LockSimpleUsedAsNestable, func );
1058 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1059 KMP_FATAL( LockStillOwned, func );
1061 __kmp_destroy_nested_ticket_lock( lck );
1070 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1072 return lck == lck->lk.initialized;
1076 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1078 return lck->lk.location;
1082 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1084 lck->lk.location = loc;
1087 static kmp_lock_flags_t
1088 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1090 return lck->lk.flags;
1094 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1096 lck->lk.flags = flags;
1155 #ifdef DEBUG_QUEUING_LOCKS
1158 #define TRACE_BUF_ELE 1024
1159 static char traces[TRACE_BUF_ELE][128] = { 0 }
1161 #define TRACE_LOCK(X,Y) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y );
1162 #define TRACE_LOCK_T(X,Y,Z) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X,Y,Z );
1163 #define TRACE_LOCK_HT(X,Y,Z,Q) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, Z, Q );
1166 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1167 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1171 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1173 i = tc % TRACE_BUF_ELE;
1174 __kmp_printf_no_lock(
"%s\n", traces[i] );
1175 i = (i+1) % TRACE_BUF_ELE;
1176 while ( i != (tc % TRACE_BUF_ELE) ) {
1177 __kmp_printf_no_lock(
"%s", traces[i] );
1178 i = (i+1) % TRACE_BUF_ELE;
1180 __kmp_printf_no_lock(
"\n" );
1182 __kmp_printf_no_lock(
1183 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1184 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1187 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1189 if ( lck->lk.head_id >= 1 ) {
1190 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1192 __kmp_printf_no_lock(
"-> %d ", t );
1193 t = __kmp_threads[t-1]->th.th_next_waiting;
1196 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1197 __kmp_printf_no_lock(
"\n\n" );
1203 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1205 return TCR_4( lck->lk.owner_id ) - 1;
1209 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1211 return lck->lk.depth_locked != -1;
1215 template <
bool takeTime>
1218 __forceinline
static void
1219 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1222 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1223 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1224 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1225 volatile kmp_uint32 *spin_here_p;
1226 kmp_int32 need_mf = 1;
1229 ompt_state_t prev_state = ompt_state_undefined;
1232 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1234 KMP_FSYNC_PREPARE( lck );
1235 KMP_DEBUG_ASSERT( this_thr != NULL );
1236 spin_here_p = & this_thr->th.th_spin_here;
1238 #ifdef DEBUG_QUEUING_LOCKS
1239 TRACE_LOCK( gtid+1,
"acq ent" );
1241 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1242 if ( this_thr->th.th_next_waiting != 0 )
1243 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1245 KMP_DEBUG_ASSERT( !*spin_here_p );
1246 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1256 *spin_here_p = TRUE;
1269 #ifdef DEBUG_QUEUING_LOCKS
1271 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1280 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1281 KMP_PACK_64( -1, 0 ),
1282 KMP_PACK_64( gtid+1, gtid+1 ) );
1283 #ifdef DEBUG_QUEUING_LOCKS
1284 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1292 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1294 #ifdef DEBUG_QUEUING_LOCKS
1295 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1304 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1306 #ifdef DEBUG_QUEUING_LOCKS
1307 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1315 kmp_int32 grabbed_lock;
1317 #ifdef DEBUG_QUEUING_LOCKS
1319 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1324 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1326 if ( grabbed_lock ) {
1328 *spin_here_p = FALSE;
1330 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1332 #ifdef DEBUG_QUEUING_LOCKS
1333 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1337 if ((ompt_status & ompt_status_track) &&
1338 prev_state != ompt_state_undefined) {
1340 this_thr->th.ompt_thread_info.state = prev_state;
1341 this_thr->th.ompt_thread_info.wait_id = 0;
1345 KMP_FSYNC_ACQUIRED( lck );
1354 if ((ompt_status & ompt_status_track) &&
1355 prev_state == ompt_state_undefined) {
1357 prev_state = this_thr->th.ompt_thread_info.state;
1358 this_thr->th.ompt_thread_info.wait_id = (uint64_t) lck;
1359 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1365 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1366 KMP_ASSERT( tail_thr != NULL );
1367 tail_thr->th.th_next_waiting = gtid+1;
1370 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1377 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1379 #ifdef DEBUG_QUEUING_LOCKS
1380 TRACE_LOCK( gtid+1,
"acq spin" );
1382 if ( this_thr->th.th_next_waiting != 0 )
1383 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1385 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1386 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1389 #ifdef DEBUG_QUEUING_LOCKS
1390 TRACE_LOCK( gtid+1,
"acq exit 2" );
1395 this_thr->th.ompt_thread_info.state = prev_state;
1396 this_thr->th.ompt_thread_info.wait_id = 0;
1406 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1408 #ifdef DEBUG_QUEUING_LOCKS
1409 TRACE_LOCK( gtid+1,
"acq retry" );
1413 KMP_ASSERT2( 0,
"should not get here" );
1417 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1419 KMP_DEBUG_ASSERT( gtid >= 0 );
1421 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1425 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1428 char const *
const func =
"omp_set_lock";
1429 if ( lck->lk.initialized != lck ) {
1430 KMP_FATAL( LockIsUninitialized, func );
1432 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1433 KMP_FATAL( LockNestableUsedAsSimple, func );
1435 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1436 KMP_FATAL( LockIsAlreadyOwned, func );
1439 __kmp_acquire_queuing_lock( lck, gtid );
1441 lck->lk.owner_id = gtid + 1;
1445 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1447 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1450 kmp_info_t *this_thr;
1453 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1454 KMP_DEBUG_ASSERT( gtid >= 0 );
1456 this_thr = __kmp_thread_from_gtid( gtid );
1457 KMP_DEBUG_ASSERT( this_thr != NULL );
1458 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1467 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1468 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1469 KMP_FSYNC_ACQUIRED(lck);
1474 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1479 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1481 char const *
const func =
"omp_test_lock";
1482 if ( lck->lk.initialized != lck ) {
1483 KMP_FATAL( LockIsUninitialized, func );
1485 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1486 KMP_FATAL( LockNestableUsedAsSimple, func );
1489 int retval = __kmp_test_queuing_lock( lck, gtid );
1492 lck->lk.owner_id = gtid + 1;
1498 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1500 register kmp_info_t *this_thr;
1501 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1502 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1504 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1505 KMP_DEBUG_ASSERT( gtid >= 0 );
1506 this_thr = __kmp_thread_from_gtid( gtid );
1507 KMP_DEBUG_ASSERT( this_thr != NULL );
1508 #ifdef DEBUG_QUEUING_LOCKS
1509 TRACE_LOCK( gtid+1,
"rel ent" );
1511 if ( this_thr->th.th_spin_here )
1512 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1513 if ( this_thr->th.th_next_waiting != 0 )
1514 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1516 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1517 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1519 KMP_FSYNC_RELEASING(lck);
1528 #ifdef DEBUG_QUEUING_LOCKS
1530 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1531 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1533 KMP_DEBUG_ASSERT( head != 0 );
1538 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1539 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1541 #ifdef DEBUG_QUEUING_LOCKS
1542 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1549 return KMP_LOCK_RELEASED;
1556 if ( head == tail ) {
1558 #ifdef DEBUG_QUEUING_LOCKS
1559 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1561 KMP_DEBUG_ASSERT( head > 0 );
1564 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1565 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1566 #ifdef DEBUG_QUEUING_LOCKS
1567 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1572 volatile kmp_int32 *waiting_id_p;
1573 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1574 KMP_DEBUG_ASSERT( head_thr != NULL );
1575 waiting_id_p = & head_thr->th.th_next_waiting;
1578 #ifdef DEBUG_QUEUING_LOCKS
1579 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1581 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1587 *head_id_p = (kmp_int32) KMP_WAIT_YIELD((
volatile kmp_uint*) waiting_id_p, 0, KMP_NEQ, NULL);
1588 #ifdef DEBUG_QUEUING_LOCKS
1589 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1596 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1597 KMP_DEBUG_ASSERT( head_thr != NULL );
1600 #ifdef DEBUG_QUEUING_LOCKS
1601 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1603 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1608 head_thr->th.th_next_waiting = 0;
1609 #ifdef DEBUG_QUEUING_LOCKS
1610 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1615 head_thr->th.th_spin_here = FALSE;
1617 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1619 #ifdef DEBUG_QUEUING_LOCKS
1620 TRACE_LOCK( gtid+1,
"rel exit 2" );
1622 return KMP_LOCK_RELEASED;
1626 #ifdef DEBUG_QUEUING_LOCKS
1627 TRACE_LOCK( gtid+1,
"rel retry" );
1631 KMP_ASSERT2( 0,
"should not get here" );
1632 return KMP_LOCK_RELEASED;
1636 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1639 char const *
const func =
"omp_unset_lock";
1641 if ( lck->lk.initialized != lck ) {
1642 KMP_FATAL( LockIsUninitialized, func );
1644 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1645 KMP_FATAL( LockNestableUsedAsSimple, func );
1647 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1648 KMP_FATAL( LockUnsettingFree, func );
1650 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1651 KMP_FATAL( LockUnsettingSetByAnother, func );
1653 lck->lk.owner_id = 0;
1654 return __kmp_release_queuing_lock( lck, gtid );
1658 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1660 lck->lk.location = NULL;
1661 lck->lk.head_id = 0;
1662 lck->lk.tail_id = 0;
1663 lck->lk.next_ticket = 0;
1664 lck->lk.now_serving = 0;
1665 lck->lk.owner_id = 0;
1666 lck->lk.depth_locked = -1;
1667 lck->lk.initialized = lck;
1669 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1673 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1675 __kmp_init_queuing_lock( lck );
1679 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1681 lck->lk.initialized = NULL;
1682 lck->lk.location = NULL;
1683 lck->lk.head_id = 0;
1684 lck->lk.tail_id = 0;
1685 lck->lk.next_ticket = 0;
1686 lck->lk.now_serving = 0;
1687 lck->lk.owner_id = 0;
1688 lck->lk.depth_locked = -1;
1692 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1694 char const *
const func =
"omp_destroy_lock";
1695 if ( lck->lk.initialized != lck ) {
1696 KMP_FATAL( LockIsUninitialized, func );
1698 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1699 KMP_FATAL( LockNestableUsedAsSimple, func );
1701 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1702 KMP_FATAL( LockStillOwned, func );
1704 __kmp_destroy_queuing_lock( lck );
1713 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1715 KMP_DEBUG_ASSERT( gtid >= 0 );
1717 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1718 lck->lk.depth_locked += 1;
1721 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1723 lck->lk.depth_locked = 1;
1725 lck->lk.owner_id = gtid + 1;
1730 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1732 char const *
const func =
"omp_set_nest_lock";
1733 if ( lck->lk.initialized != lck ) {
1734 KMP_FATAL( LockIsUninitialized, func );
1736 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1737 KMP_FATAL( LockSimpleUsedAsNestable, func );
1739 __kmp_acquire_nested_queuing_lock( lck, gtid );
1743 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1747 KMP_DEBUG_ASSERT( gtid >= 0 );
1749 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1750 retval = ++lck->lk.depth_locked;
1752 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1757 retval = lck->lk.depth_locked = 1;
1759 lck->lk.owner_id = gtid + 1;
1765 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1768 char const *
const func =
"omp_test_nest_lock";
1769 if ( lck->lk.initialized != lck ) {
1770 KMP_FATAL( LockIsUninitialized, func );
1772 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1773 KMP_FATAL( LockSimpleUsedAsNestable, func );
1775 return __kmp_test_nested_queuing_lock( lck, gtid );
1779 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1781 KMP_DEBUG_ASSERT( gtid >= 0 );
1784 if ( --(lck->lk.depth_locked) == 0 ) {
1786 lck->lk.owner_id = 0;
1787 __kmp_release_queuing_lock( lck, gtid );
1788 return KMP_LOCK_RELEASED;
1790 return KMP_LOCK_STILL_HELD;
1794 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1796 char const *
const func =
"omp_unset_nest_lock";
1798 if ( lck->lk.initialized != lck ) {
1799 KMP_FATAL( LockIsUninitialized, func );
1801 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1802 KMP_FATAL( LockSimpleUsedAsNestable, func );
1804 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1805 KMP_FATAL( LockUnsettingFree, func );
1807 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1808 KMP_FATAL( LockUnsettingSetByAnother, func );
1810 return __kmp_release_nested_queuing_lock( lck, gtid );
1814 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1816 __kmp_init_queuing_lock( lck );
1817 lck->lk.depth_locked = 0;
1821 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1823 __kmp_init_nested_queuing_lock( lck );
1827 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1829 __kmp_destroy_queuing_lock( lck );
1830 lck->lk.depth_locked = 0;
1834 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1836 char const *
const func =
"omp_destroy_nest_lock";
1837 if ( lck->lk.initialized != lck ) {
1838 KMP_FATAL( LockIsUninitialized, func );
1840 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1841 KMP_FATAL( LockSimpleUsedAsNestable, func );
1843 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1844 KMP_FATAL( LockStillOwned, func );
1846 __kmp_destroy_nested_queuing_lock( lck );
1855 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1857 return lck == lck->lk.initialized;
1861 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1863 return lck->lk.location;
1867 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1869 lck->lk.location = loc;
1872 static kmp_lock_flags_t
1873 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1875 return lck->lk.flags;
1879 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1881 lck->lk.flags = flags;
1884 #if KMP_USE_ADAPTIVE_LOCKS
1894 #define _XBEGIN_STARTED (~0u)
1895 #define _XABORT_EXPLICIT (1 << 0)
1896 #define _XABORT_RETRY (1 << 1)
1897 #define _XABORT_CONFLICT (1 << 2)
1898 #define _XABORT_CAPACITY (1 << 3)
1899 #define _XABORT_DEBUG (1 << 4)
1900 #define _XABORT_NESTED (1 << 5)
1901 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1904 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1906 #define STRINGIZE_INTERNAL(arg) #arg
1907 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1915 static __inline
int _xbegin()
1944 #endif // KMP_ARCH_X86_64
1954 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n"
1957 "1: movl %%eax,%0\n"
1959 :
"+r"(res)::
"memory",
"%eax");
1960 #endif // KMP_OS_WINDOWS
1967 static __inline
void _xend()
1976 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1986 #define _xabort(ARG) \
1991 #define _xabort(ARG) \
1992 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory");
1998 #if KMP_DEBUG_ADAPTIVE_LOCKS
2003 static kmp_adaptive_lock_statistics_t destroyedStats;
2006 static kmp_adaptive_lock_info_t liveLocks;
2009 static kmp_bootstrap_lock_t chain_lock;
2013 __kmp_init_speculative_stats()
2015 kmp_adaptive_lock_info_t *lck = &liveLocks;
2017 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
2018 lck->stats.next = lck;
2019 lck->stats.prev = lck;
2021 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2022 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2024 __kmp_init_bootstrap_lock( &chain_lock );
2030 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
2032 __kmp_acquire_bootstrap_lock( &chain_lock );
2034 lck->stats.next = liveLocks.stats.next;
2035 lck->stats.prev = &liveLocks;
2037 liveLocks.stats.next = lck;
2038 lck->stats.next->stats.prev = lck;
2040 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2041 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2043 __kmp_release_bootstrap_lock( &chain_lock );
2047 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2049 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2050 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2052 kmp_adaptive_lock_info_t * n = lck->stats.next;
2053 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2060 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2062 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2063 __kmp_remember_lock( lck );
2067 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2069 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2071 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2072 t->successfulSpeculations += s->successfulSpeculations;
2073 t->hardFailedSpeculations += s->hardFailedSpeculations;
2074 t->softFailedSpeculations += s->softFailedSpeculations;
2075 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2076 t->lemmingYields += s->lemmingYields;
2080 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2082 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2084 __kmp_acquire_bootstrap_lock( &chain_lock );
2086 __kmp_add_stats( &destroyedStats, lck );
2087 __kmp_forget_lock( lck );
2089 __kmp_release_bootstrap_lock( &chain_lock );
2093 percent (kmp_uint32 count, kmp_uint32 total)
2095 return (total == 0) ? 0.0: (100.0 * count)/total;
2099 FILE * __kmp_open_stats_file()
2101 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2104 size_t buffLen = KMP_STRLEN( __kmp_speculative_statsfile ) + 20;
2105 char buffer[buffLen];
2106 KMP_SNPRINTF (&buffer[0], buffLen, __kmp_speculative_statsfile,
2107 (kmp_int32)getpid());
2108 FILE * result = fopen(&buffer[0],
"w");
2111 return result ? result : stdout;
2115 __kmp_print_speculative_stats()
2117 if (__kmp_user_lock_kind != lk_adaptive)
2120 FILE * statsFile = __kmp_open_stats_file();
2122 kmp_adaptive_lock_statistics_t total = destroyedStats;
2123 kmp_adaptive_lock_info_t *lck;
2125 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2126 __kmp_add_stats( &total, lck );
2128 kmp_adaptive_lock_statistics_t *t = &total;
2129 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2130 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2131 t->softFailedSpeculations;
2133 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2134 fprintf ( statsFile,
" Lock parameters: \n"
2135 " max_soft_retries : %10d\n"
2136 " max_badness : %10d\n",
2137 __kmp_adaptive_backoff_params.max_soft_retries,
2138 __kmp_adaptive_backoff_params.max_badness);
2139 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2140 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2141 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2142 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2143 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2144 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2145 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2147 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2148 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2149 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2150 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2151 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2152 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2153 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2155 if (statsFile != stdout)
2156 fclose( statsFile );
2159 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ )
2161 # define KMP_INC_STAT(lck,stat)
2163 #endif // KMP_DEBUG_ADAPTIVE_LOCKS
2166 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2170 bool res = lck->lk.head_id == 0;
2174 #if KMP_COMPILER_ICC
2177 __sync_synchronize();
2184 static __inline
void
2185 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2188 lck->lk.adaptive.badness = 0;
2189 KMP_INC_STAT(lck,successfulSpeculations);
2193 static __inline
void
2194 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2196 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2197 if ( newBadness > lck->lk.adaptive.max_badness) {
2200 lck->lk.adaptive.badness = newBadness;
2206 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2208 kmp_uint32 badness = lck->lk.adaptive.badness;
2209 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2210 int res = (attempts & badness) == 0;
2218 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2220 int retries = lck->lk.adaptive.max_soft_retries;
2228 kmp_uint32 status = _xbegin();
2233 if (status == _XBEGIN_STARTED )
2239 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2245 KMP_ASSERT2( 0, "should not get here" );
2250 if ( status & SOFT_ABORT_MASK)
2252 KMP_INC_STAT(lck,softFailedSpeculations);
2257 KMP_INC_STAT(lck,hardFailedSpeculations);
2262 }
while( retries-- );
2266 __kmp_step_badness( lck );
2274 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2277 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2282 lck->lk.adaptive.acquire_attempts++;
2285 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2287 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2297 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2299 char const *
const func =
"omp_test_lock";
2300 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2301 KMP_FATAL( LockIsUninitialized, func );
2304 int retval = __kmp_test_adaptive_lock( lck, gtid );
2307 lck->lk.qlk.owner_id = gtid + 1;
2327 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2329 if ( __kmp_should_speculate( lck, gtid ) )
2331 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2333 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2345 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2347 KMP_INC_STAT(lck,lemmingYields);
2351 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2358 lck->lk.adaptive.acquire_attempts++;
2360 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2362 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2366 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2368 char const *
const func =
"omp_set_lock";
2369 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2370 KMP_FATAL( LockIsUninitialized, func );
2372 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2373 KMP_FATAL( LockIsAlreadyOwned, func );
2376 __kmp_acquire_adaptive_lock( lck, gtid );
2378 lck->lk.qlk.owner_id = gtid + 1;
2382 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2384 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2389 __kmp_update_badness_after_success( lck );
2394 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2396 return KMP_LOCK_RELEASED;
2400 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2402 char const *
const func =
"omp_unset_lock";
2404 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2405 KMP_FATAL( LockIsUninitialized, func );
2407 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2408 KMP_FATAL( LockUnsettingFree, func );
2410 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2411 KMP_FATAL( LockUnsettingSetByAnother, func );
2413 lck->lk.qlk.owner_id = 0;
2414 __kmp_release_adaptive_lock( lck, gtid );
2415 return KMP_LOCK_RELEASED;
2419 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2421 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2422 lck->lk.adaptive.badness = 0;
2423 lck->lk.adaptive.acquire_attempts = 0;
2424 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2425 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2426 #if KMP_DEBUG_ADAPTIVE_LOCKS
2427 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2429 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2433 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2435 __kmp_init_adaptive_lock( lck );
2439 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2441 #if KMP_DEBUG_ADAPTIVE_LOCKS
2442 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2444 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2449 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2451 char const *
const func =
"omp_destroy_lock";
2452 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2453 KMP_FATAL( LockIsUninitialized, func );
2455 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2456 KMP_FATAL( LockStillOwned, func );
2458 __kmp_destroy_adaptive_lock( lck );
2462 #endif // KMP_USE_ADAPTIVE_LOCKS
2470 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2472 return TCR_4( lck->lk.owner_id ) - 1;
2476 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2478 return lck->lk.depth_locked != -1;
2481 __forceinline
static void
2482 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2484 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2485 kmp_uint64 mask = TCR_8(lck->lk.mask);
2486 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2487 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2488 TCR_PTR(lck->lk.polls);
2490 #ifdef USE_LOCK_PROFILE
2491 if (TCR_8(polls[ticket & mask].poll) != ticket)
2492 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2508 KMP_FSYNC_PREPARE(lck);
2509 KMP_INIT_YIELD(spins);
2510 while (TCR_8(polls[ticket & mask]).poll < ticket) {
2515 KMP_YIELD(TCR_4(__kmp_nth)
2516 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2517 KMP_YIELD_SPIN(spins);
2528 mask = TCR_8(lck->lk.mask);
2529 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2530 TCR_PTR(lck->lk.polls);
2536 KMP_FSYNC_ACQUIRED(lck);
2537 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2539 lck->lk.now_serving = ticket;
2548 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2549 __kmp_free((
void *)lck->lk.old_polls);
2550 lck->lk.old_polls = NULL;
2551 lck->lk.cleanup_ticket = 0;
2559 if (lck->lk.old_polls == NULL) {
2560 bool reconfigure =
false;
2561 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2562 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2564 if (TCR_4(__kmp_nth)
2565 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2570 if (num_polls > 1) {
2572 num_polls = TCR_4(lck->lk.num_polls);
2575 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2576 __kmp_allocate(num_polls *
sizeof(*polls));
2577 polls[0].poll = ticket;
2586 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2587 if (num_waiting > num_polls) {
2588 kmp_uint32 old_num_polls = num_polls;
2591 mask = (mask << 1) | 1;
2593 }
while (num_polls <= num_waiting);
2601 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2602 __kmp_allocate(num_polls *
sizeof(*polls));
2604 for (i = 0; i < old_num_polls; i++) {
2605 polls[i].poll = old_polls[i].poll;
2622 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2623 ticket, lck, num_polls));
2625 lck->lk.old_polls = old_polls;
2626 lck->lk.polls = polls;
2630 lck->lk.num_polls = num_polls;
2631 lck->lk.mask = mask;
2641 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2647 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2649 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2653 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2655 char const *
const func =
"omp_set_lock";
2656 if ( lck->lk.initialized != lck ) {
2657 KMP_FATAL( LockIsUninitialized, func );
2659 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2660 KMP_FATAL( LockNestableUsedAsSimple, func );
2662 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2663 KMP_FATAL( LockIsAlreadyOwned, func );
2666 __kmp_acquire_drdpa_lock( lck, gtid );
2668 lck->lk.owner_id = gtid + 1;
2672 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2678 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2679 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2680 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2681 TCR_PTR(lck->lk.polls);
2682 kmp_uint64 mask = TCR_8(lck->lk.mask);
2683 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2684 kmp_uint64 next_ticket = ticket + 1;
2685 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2686 ticket, next_ticket)) {
2687 KMP_FSYNC_ACQUIRED(lck);
2688 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2690 lck->lk.now_serving = ticket;
2708 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2710 char const *
const func =
"omp_test_lock";
2711 if ( lck->lk.initialized != lck ) {
2712 KMP_FATAL( LockIsUninitialized, func );
2714 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2715 KMP_FATAL( LockNestableUsedAsSimple, func );
2718 int retval = __kmp_test_drdpa_lock( lck, gtid );
2721 lck->lk.owner_id = gtid + 1;
2727 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2734 kmp_uint64 ticket = lck->lk.now_serving + 1;
2735 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2736 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2737 TCR_PTR(lck->lk.polls);
2738 kmp_uint64 mask = TCR_8(lck->lk.mask);
2739 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2741 KMP_FSYNC_RELEASING(lck);
2742 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2743 return KMP_LOCK_RELEASED;
2747 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2749 char const *
const func =
"omp_unset_lock";
2751 if ( lck->lk.initialized != lck ) {
2752 KMP_FATAL( LockIsUninitialized, func );
2754 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2755 KMP_FATAL( LockNestableUsedAsSimple, func );
2757 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2758 KMP_FATAL( LockUnsettingFree, func );
2760 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2761 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2762 KMP_FATAL( LockUnsettingSetByAnother, func );
2764 lck->lk.owner_id = 0;
2765 return __kmp_release_drdpa_lock( lck, gtid );
2769 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2771 lck->lk.location = NULL;
2773 lck->lk.num_polls = 1;
2774 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2775 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2776 lck->lk.cleanup_ticket = 0;
2777 lck->lk.old_polls = NULL;
2778 lck->lk.next_ticket = 0;
2779 lck->lk.now_serving = 0;
2780 lck->lk.owner_id = 0;
2781 lck->lk.depth_locked = -1;
2782 lck->lk.initialized = lck;
2784 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2788 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2790 __kmp_init_drdpa_lock( lck );
2794 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2796 lck->lk.initialized = NULL;
2797 lck->lk.location = NULL;
2798 if (lck->lk.polls != NULL) {
2799 __kmp_free((
void *)lck->lk.polls);
2800 lck->lk.polls = NULL;
2802 if (lck->lk.old_polls != NULL) {
2803 __kmp_free((
void *)lck->lk.old_polls);
2804 lck->lk.old_polls = NULL;
2807 lck->lk.num_polls = 0;
2808 lck->lk.cleanup_ticket = 0;
2809 lck->lk.next_ticket = 0;
2810 lck->lk.now_serving = 0;
2811 lck->lk.owner_id = 0;
2812 lck->lk.depth_locked = -1;
2816 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2818 char const *
const func =
"omp_destroy_lock";
2819 if ( lck->lk.initialized != lck ) {
2820 KMP_FATAL( LockIsUninitialized, func );
2822 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2823 KMP_FATAL( LockNestableUsedAsSimple, func );
2825 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2826 KMP_FATAL( LockStillOwned, func );
2828 __kmp_destroy_drdpa_lock( lck );
2837 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2839 KMP_DEBUG_ASSERT( gtid >= 0 );
2841 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2842 lck->lk.depth_locked += 1;
2845 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2847 lck->lk.depth_locked = 1;
2849 lck->lk.owner_id = gtid + 1;
2854 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2856 char const *
const func =
"omp_set_nest_lock";
2857 if ( lck->lk.initialized != lck ) {
2858 KMP_FATAL( LockIsUninitialized, func );
2860 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2861 KMP_FATAL( LockSimpleUsedAsNestable, func );
2863 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2867 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2871 KMP_DEBUG_ASSERT( gtid >= 0 );
2873 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2874 retval = ++lck->lk.depth_locked;
2876 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2881 retval = lck->lk.depth_locked = 1;
2883 lck->lk.owner_id = gtid + 1;
2889 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2891 char const *
const func =
"omp_test_nest_lock";
2892 if ( lck->lk.initialized != lck ) {
2893 KMP_FATAL( LockIsUninitialized, func );
2895 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2896 KMP_FATAL( LockSimpleUsedAsNestable, func );
2898 return __kmp_test_nested_drdpa_lock( lck, gtid );
2902 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2904 KMP_DEBUG_ASSERT( gtid >= 0 );
2907 if ( --(lck->lk.depth_locked) == 0 ) {
2909 lck->lk.owner_id = 0;
2910 __kmp_release_drdpa_lock( lck, gtid );
2911 return KMP_LOCK_RELEASED;
2913 return KMP_LOCK_STILL_HELD;
2917 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2919 char const *
const func =
"omp_unset_nest_lock";
2921 if ( lck->lk.initialized != lck ) {
2922 KMP_FATAL( LockIsUninitialized, func );
2924 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2925 KMP_FATAL( LockSimpleUsedAsNestable, func );
2927 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2928 KMP_FATAL( LockUnsettingFree, func );
2930 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2931 KMP_FATAL( LockUnsettingSetByAnother, func );
2933 return __kmp_release_nested_drdpa_lock( lck, gtid );
2937 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2939 __kmp_init_drdpa_lock( lck );
2940 lck->lk.depth_locked = 0;
2944 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2946 __kmp_init_nested_drdpa_lock( lck );
2950 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2952 __kmp_destroy_drdpa_lock( lck );
2953 lck->lk.depth_locked = 0;
2957 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2959 char const *
const func =
"omp_destroy_nest_lock";
2960 if ( lck->lk.initialized != lck ) {
2961 KMP_FATAL( LockIsUninitialized, func );
2963 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2964 KMP_FATAL( LockSimpleUsedAsNestable, func );
2966 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2967 KMP_FATAL( LockStillOwned, func );
2969 __kmp_destroy_nested_drdpa_lock( lck );
2978 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
2980 return lck == lck->lk.initialized;
2984 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
2986 return lck->lk.location;
2990 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
2992 lck->lk.location = loc;
2995 static kmp_lock_flags_t
2996 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
2998 return lck->lk.flags;
3002 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
3004 lck->lk.flags = flags;
3007 #if KMP_USE_DYNAMIC_LOCK
3011 typedef enum kmp_lock_hint_t {
3012 kmp_lock_hint_none = 0,
3013 kmp_lock_hint_contended,
3014 kmp_lock_hint_uncontended,
3015 kmp_lock_hint_nonspeculative,
3016 kmp_lock_hint_speculative,
3017 kmp_lock_hint_adaptive,
3022 #define expand_init_lock(l, a) \
3023 static void init_##l##_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq) { \
3024 *lck = DYNA_LOCK_FREE(l); \
3025 KA_TRACE(20, ("Initialized direct lock, tag = %x\n", *lck)); \
3027 FOREACH_D_LOCK(expand_init_lock, 0)
3028 #undef expand_init_lock
3034 # define machine_pause() _mm_delay_32(10) // TODO: find the right argument
3036 # define machine_pause() _mm_pause()
3038 #define HLE_ACQUIRE ".byte 0xf2;"
3039 #define HLE_RELEASE ".byte 0xf3;"
3041 static inline kmp_uint32
3042 swap4(kmp_uint32
volatile *p, kmp_uint32 v)
3044 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0"
3052 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
3058 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3061 if (swap4(lck, DYNA_LOCK_BUSY(1, hle)) != DYNA_LOCK_FREE(hle)) {
3064 while (*(kmp_uint32
volatile *)lck != DYNA_LOCK_FREE(hle)) {
3065 for (
int i = delay; i != 0; --i)
3067 delay = ((delay << 1) | 1) & 7;
3069 }
while (swap4(lck, DYNA_LOCK_BUSY(1, hle)) != DYNA_LOCK_FREE(hle));
3074 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3076 __kmp_acquire_hle_lock(lck, gtid);
3080 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3082 __asm__
volatile(HLE_RELEASE
"movl %1,%0"
3084 :
"r"(DYNA_LOCK_FREE(hle))
3089 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3091 __kmp_release_hle_lock(lck, gtid);
3095 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3097 return swap4(lck, DYNA_LOCK_BUSY(1, hle)) == DYNA_LOCK_FREE(hle);
3101 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3103 return __kmp_test_hle_lock(lck, gtid);
3106 #endif // DYNA_HAS_HLE
3109 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
3110 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
3111 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3112 static void __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3113 static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3114 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3115 static void __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3116 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3122 #define expand_func0(l, op) 0,op##_##l##_##lock,
3123 void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
3124 = { __kmp_init_indirect_lock, 0, FOREACH_D_LOCK(expand_func0, init) };
3126 #define expand_func1(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_##lock,
3127 void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *)
3128 = { __kmp_destroy_indirect_lock, 0, FOREACH_D_LOCK(expand_func1, destroy) };
3131 #define expand_func2(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
3132 #define expand_func2c(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3133 static void (*direct_set_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3134 = { { __kmp_set_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, acquire) },
3135 { __kmp_set_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, acquire) } };
3136 static void (*direct_unset_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3137 = { { __kmp_unset_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, release) },
3138 { __kmp_unset_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, release) } };
3140 #define expand_func3(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
3141 #define expand_func3c(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3142 static int (*direct_test_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3143 = { { __kmp_test_indirect_lock, 0, FOREACH_D_LOCK(expand_func3, test) },
3144 { __kmp_test_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func3c, test) } };
3147 void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3148 void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3149 int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3154 #define expand_func4(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock,
3155 void (*__kmp_indirect_init_ops[])(kmp_user_lock_p)
3156 = { FOREACH_I_LOCK(expand_func4, init) };
3157 void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p)
3158 = { FOREACH_I_LOCK(expand_func4, destroy) };
3161 #define expand_func5(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
3162 #define expand_func5c(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3163 static void (*indirect_set_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3164 = { { FOREACH_I_LOCK(expand_func5, acquire) },
3165 { FOREACH_I_LOCK(expand_func5c, acquire) } };
3166 static void (*indirect_unset_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3167 = { { FOREACH_I_LOCK(expand_func5, release) },
3168 { FOREACH_I_LOCK(expand_func5c, release) } };
3170 #define expand_func6(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
3171 #define expand_func6c(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3172 static int (*indirect_test_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3173 = { { FOREACH_I_LOCK(expand_func6, test) },
3174 { FOREACH_I_LOCK(expand_func6c, test) } };
3177 void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32) = 0;
3178 void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32) = 0;
3179 int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32) = 0;
3182 kmp_indirect_lock_t **__kmp_indirect_lock_table;
3183 kmp_lock_index_t __kmp_indirect_lock_table_size;
3184 kmp_lock_index_t __kmp_indirect_lock_table_next;
3187 static kmp_uint32 __kmp_indirect_lock_size[DYNA_NUM_I_LOCKS] = {
3188 sizeof(kmp_ticket_lock_t),
sizeof(kmp_queuing_lock_t),
3189 #if KMP_USE_ADAPTIVE_LOCKS
3190 sizeof(kmp_adaptive_lock_t),
3192 sizeof(kmp_drdpa_lock_t),
3193 sizeof(kmp_tas_lock_t),
3195 sizeof(kmp_futex_lock_t),
3197 sizeof(kmp_ticket_lock_t),
sizeof(kmp_queuing_lock_t),
3198 sizeof(kmp_drdpa_lock_t)
3202 void (*__kmp_indirect_set_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *) = { 0 };
3203 void (*__kmp_indirect_set_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t) = { 0 };
3204 const ident_t * (*__kmp_indirect_get_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3205 kmp_lock_flags_t (*__kmp_indirect_get_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3208 static kmp_indirect_lock_t * __kmp_indirect_lock_pool[DYNA_NUM_I_LOCKS] = { 0 };
3212 __kmp_insert_indirect_lock(kmp_indirect_lock_t *lck)
3214 kmp_lock_index_t next = __kmp_indirect_lock_table_next;
3216 if (next >= __kmp_indirect_lock_table_size) {
3218 kmp_lock_index_t size = __kmp_indirect_lock_table_size;
3219 kmp_indirect_lock_t **old_table = __kmp_indirect_lock_table;
3220 __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(2*next*
sizeof(kmp_indirect_lock_t *));
3221 KMP_MEMCPY(__kmp_indirect_lock_table, old_table, next*
sizeof(kmp_indirect_lock_t *));
3222 __kmp_free(old_table);
3223 __kmp_indirect_lock_table_size = 2*next;
3226 __kmp_indirect_lock_table[next] = lck;
3227 __kmp_indirect_lock_table_next++;
3232 kmp_indirect_lock_t *
3233 __kmp_allocate_indirect_lock(
void **user_lock, kmp_int32 gtid, kmp_indirect_locktag_t tag)
3235 kmp_indirect_lock_t *lck;
3236 kmp_lock_index_t idx;
3238 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3240 if (__kmp_indirect_lock_pool[tag] != NULL) {
3241 lck = __kmp_indirect_lock_pool[tag];
3242 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3243 idx = lck->lock->pool.index;
3244 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3246 lck = (kmp_indirect_lock_t *)__kmp_allocate(
sizeof(kmp_indirect_lock_t));
3247 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3248 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3249 idx = __kmp_insert_indirect_lock(lck);
3252 __kmp_release_lock(&__kmp_global_lock, gtid);
3256 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3257 *((kmp_lock_index_t *)user_lock) = idx << 1;
3259 *((kmp_indirect_lock_t **)user_lock) = lck;
3266 static __forceinline
3267 kmp_indirect_lock_t *
3268 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func)
3270 if (__kmp_env_consistency_check) {
3271 kmp_indirect_lock_t *lck = NULL;
3272 if (user_lock == NULL) {
3273 KMP_FATAL(LockIsUninitialized, func);
3275 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3276 kmp_lock_index_t idx = DYNA_EXTRACT_I_INDEX(user_lock);
3277 if (idx < 0 || idx >= __kmp_indirect_lock_table_size) {
3278 KMP_FATAL(LockIsUninitialized, func);
3280 lck = __kmp_indirect_lock_table[idx];
3282 lck = *((kmp_indirect_lock_t **)user_lock);
3285 KMP_FATAL(LockIsUninitialized, func);
3289 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3290 return __kmp_indirect_lock_table[DYNA_EXTRACT_I_INDEX(user_lock)];
3292 return *((kmp_indirect_lock_t **)user_lock);
3298 __kmp_init_indirect_lock(kmp_dyna_lock_t * lock, kmp_dyna_lockseq_t seq)
3300 #if KMP_USE_ADAPTIVE_LOCKS
3301 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3302 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3303 seq = lockseq_queuing;
3306 kmp_indirect_locktag_t tag = DYNA_GET_I_TAG(seq);
3307 kmp_indirect_lock_t *l = __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3308 DYNA_I_LOCK_FUNC(l, init)(l->lock);
3309 KA_TRACE(20, (
"__kmp_init_indirect_lock: initialized indirect lock, tag = %x\n", l->type));
3313 __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock)
3315 kmp_uint32 gtid = __kmp_entry_gtid();
3316 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3317 DYNA_I_LOCK_FUNC(l, destroy)(l->lock);
3318 kmp_indirect_locktag_t tag = l->type;
3320 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3323 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3324 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3325 l->lock->pool.index = DYNA_EXTRACT_I_INDEX(lock);
3327 __kmp_indirect_lock_pool[tag] = l;
3329 __kmp_release_lock(&__kmp_global_lock, gtid);
3333 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3335 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3336 DYNA_I_LOCK_FUNC(l, set)(l->lock, gtid);
3340 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3342 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3343 DYNA_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3347 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3349 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3350 return DYNA_I_LOCK_FUNC(l, test)(l->lock, gtid);
3354 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3356 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3357 DYNA_I_LOCK_FUNC(l, set)(l->lock, gtid);
3361 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3363 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3364 DYNA_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3368 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3370 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3371 return DYNA_I_LOCK_FUNC(l, test)(l->lock, gtid);
3374 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3378 __kmp_init_lock_hinted(
void **lock,
int hint)
3380 kmp_dyna_lockseq_t seq;
3382 case kmp_lock_hint_uncontended:
3385 case kmp_lock_hint_speculative:
3392 case kmp_lock_hint_adaptive:
3393 #if KMP_USE_ADAPTIVE_LOCKS
3394 seq = lockseq_adaptive;
3396 seq = lockseq_queuing;
3400 case kmp_lock_hint_contended:
3401 case kmp_lock_hint_nonspeculative:
3403 seq = lockseq_queuing;
3406 if (DYNA_IS_D_LOCK(seq)) {
3407 DYNA_INIT_D_LOCK(lock, seq);
3409 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
3412 DYNA_INIT_I_LOCK(lock, seq);
3414 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(lock);
3415 __kmp_itt_lock_creating(ilk->lock, NULL);
3422 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
3426 case lockseq_nested_tas:
3427 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3430 case lockseq_nested_futex:
3431 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3433 case lockseq_ticket:
3434 case lockseq_nested_ticket:
3435 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3436 case lockseq_queuing:
3437 case lockseq_nested_queuing:
3438 #if KMP_USE_ADAPTIVE_LOCKS
3439 case lockseq_adaptive:
3440 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3443 case lockseq_nested_drdpa:
3444 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3453 __kmp_init_nest_lock_hinted(
void **lock,
int hint)
3455 kmp_dyna_lockseq_t seq;
3457 case kmp_lock_hint_uncontended:
3458 seq = lockseq_nested_tas;
3461 case kmp_lock_hint_contended:
3462 case kmp_lock_hint_nonspeculative:
3464 seq = lockseq_nested_queuing;
3467 DYNA_INIT_I_LOCK(lock, seq);
3469 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(lock);
3470 __kmp_itt_lock_creating(ilk->lock, NULL);
3476 __kmp_init_indirect_lock_table()
3478 __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *)*1024);
3479 __kmp_indirect_lock_table_size = 1024;
3480 __kmp_indirect_lock_table_next = 0;
3483 #if KMP_USE_ADAPTIVE_LOCKS
3484 # define init_lock_func(table, expand) { \
3485 table[locktag_ticket] = expand(ticket); \
3486 table[locktag_queuing] = expand(queuing); \
3487 table[locktag_adaptive] = expand(queuing); \
3488 table[locktag_drdpa] = expand(drdpa); \
3489 table[locktag_nested_ticket] = expand(ticket); \
3490 table[locktag_nested_queuing] = expand(queuing); \
3491 table[locktag_nested_drdpa] = expand(drdpa); \
3494 # define init_lock_func(table, expand) { \
3495 table[locktag_ticket] = expand(ticket); \
3496 table[locktag_queuing] = expand(queuing); \
3497 table[locktag_drdpa] = expand(drdpa); \
3498 table[locktag_nested_ticket] = expand(ticket); \
3499 table[locktag_nested_queuing] = expand(queuing); \
3500 table[locktag_nested_drdpa] = expand(drdpa); \
3502 #endif // KMP_USE_ADAPTIVE_LOCKS
3506 __kmp_init_dynamic_user_locks()
3509 int offset = (__kmp_env_consistency_check)? 1: 0;
3510 __kmp_direct_set_ops = direct_set_tab[offset];
3511 __kmp_direct_unset_ops = direct_unset_tab[offset];
3512 __kmp_direct_test_ops = direct_test_tab[offset];
3513 __kmp_indirect_set_ops = indirect_set_tab[offset];
3514 __kmp_indirect_unset_ops = indirect_unset_tab[offset];
3515 __kmp_indirect_test_ops = indirect_test_tab[offset];
3516 __kmp_init_indirect_lock_table();
3520 #define expand_func(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location
3521 init_lock_func(__kmp_indirect_set_location, expand_func);
3523 #define expand_func(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags
3524 init_lock_func(__kmp_indirect_set_flags, expand_func);
3526 #define expand_func(l) (const ident_t * (*)(kmp_user_lock_p))__kmp_get_##l##_lock_location
3527 init_lock_func(__kmp_indirect_get_location, expand_func);
3529 #define expand_func(l) (kmp_lock_flags_t (*)(kmp_user_lock_p))__kmp_get_##l##_lock_flags
3530 init_lock_func(__kmp_indirect_get_flags, expand_func);
3533 __kmp_init_user_locks = TRUE;
3538 __kmp_cleanup_indirect_user_locks()
3544 for (k = 0; k < DYNA_NUM_I_LOCKS; ++k) {
3545 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3547 kmp_indirect_lock_t *ll = l;
3548 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3549 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3550 __kmp_indirect_lock_table[ll->lock->pool.index] = NULL;
3552 __kmp_free(ll->lock);
3557 for (i = 0; i < __kmp_indirect_lock_table_next; i++) {
3558 kmp_indirect_lock_t *l = __kmp_indirect_lock_table[i];
3561 DYNA_I_LOCK_FUNC(l, destroy)(l->lock);
3562 __kmp_free(l->lock);
3567 __kmp_free(__kmp_indirect_lock_table);
3569 __kmp_init_user_locks = FALSE;
3572 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3573 int __kmp_num_locks_in_block = 1;
3575 #else // KMP_USE_DYNAMIC_LOCK
3584 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3586 size_t __kmp_base_user_lock_size = 0;
3587 size_t __kmp_user_lock_size = 0;
3589 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3590 void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3592 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3593 int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3594 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3595 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3596 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3597 void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3599 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3600 int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3601 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3602 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3604 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3605 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3606 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3607 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3608 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3610 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3612 switch ( user_lock_kind ) {
3618 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3619 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3621 __kmp_get_user_lock_owner_ =
3622 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3623 ( &__kmp_get_tas_lock_owner );
3625 if ( __kmp_env_consistency_check ) {
3626 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3627 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3630 KMP_BIND_USER_LOCK(tas);
3631 KMP_BIND_NESTED_USER_LOCK(tas);
3634 __kmp_destroy_user_lock_ =
3635 ( void ( * )( kmp_user_lock_p ) )
3636 ( &__kmp_destroy_tas_lock );
3638 __kmp_is_user_lock_initialized_ =
3639 ( int ( * )( kmp_user_lock_p ) ) NULL;
3641 __kmp_get_user_lock_location_ =
3642 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3644 __kmp_set_user_lock_location_ =
3645 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3647 __kmp_get_user_lock_flags_ =
3648 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3650 __kmp_set_user_lock_flags_ =
3651 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3655 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
3658 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3659 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3661 __kmp_get_user_lock_owner_ =
3662 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3663 ( &__kmp_get_futex_lock_owner );
3665 if ( __kmp_env_consistency_check ) {
3666 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3667 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3670 KMP_BIND_USER_LOCK(futex);
3671 KMP_BIND_NESTED_USER_LOCK(futex);
3674 __kmp_destroy_user_lock_ =
3675 ( void ( * )( kmp_user_lock_p ) )
3676 ( &__kmp_destroy_futex_lock );
3678 __kmp_is_user_lock_initialized_ =
3679 ( int ( * )( kmp_user_lock_p ) ) NULL;
3681 __kmp_get_user_lock_location_ =
3682 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3684 __kmp_set_user_lock_location_ =
3685 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3687 __kmp_get_user_lock_flags_ =
3688 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3690 __kmp_set_user_lock_flags_ =
3691 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3695 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
3698 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3699 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3701 __kmp_get_user_lock_owner_ =
3702 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3703 ( &__kmp_get_ticket_lock_owner );
3705 if ( __kmp_env_consistency_check ) {
3706 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3707 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3710 KMP_BIND_USER_LOCK(ticket);
3711 KMP_BIND_NESTED_USER_LOCK(ticket);
3714 __kmp_destroy_user_lock_ =
3715 ( void ( * )( kmp_user_lock_p ) )
3716 ( &__kmp_destroy_ticket_lock );
3718 __kmp_is_user_lock_initialized_ =
3719 ( int ( * )( kmp_user_lock_p ) )
3720 ( &__kmp_is_ticket_lock_initialized );
3722 __kmp_get_user_lock_location_ =
3723 (
const ident_t * ( * )( kmp_user_lock_p ) )
3724 ( &__kmp_get_ticket_lock_location );
3726 __kmp_set_user_lock_location_ =
3727 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3728 ( &__kmp_set_ticket_lock_location );
3730 __kmp_get_user_lock_flags_ =
3731 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3732 ( &__kmp_get_ticket_lock_flags );
3734 __kmp_set_user_lock_flags_ =
3735 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3736 ( &__kmp_set_ticket_lock_flags );
3741 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3742 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3744 __kmp_get_user_lock_owner_ =
3745 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3746 ( &__kmp_get_queuing_lock_owner );
3748 if ( __kmp_env_consistency_check ) {
3749 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3750 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3753 KMP_BIND_USER_LOCK(queuing);
3754 KMP_BIND_NESTED_USER_LOCK(queuing);
3757 __kmp_destroy_user_lock_ =
3758 ( void ( * )( kmp_user_lock_p ) )
3759 ( &__kmp_destroy_queuing_lock );
3761 __kmp_is_user_lock_initialized_ =
3762 ( int ( * )( kmp_user_lock_p ) )
3763 ( &__kmp_is_queuing_lock_initialized );
3765 __kmp_get_user_lock_location_ =
3766 (
const ident_t * ( * )( kmp_user_lock_p ) )
3767 ( &__kmp_get_queuing_lock_location );
3769 __kmp_set_user_lock_location_ =
3770 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3771 ( &__kmp_set_queuing_lock_location );
3773 __kmp_get_user_lock_flags_ =
3774 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3775 ( &__kmp_get_queuing_lock_flags );
3777 __kmp_set_user_lock_flags_ =
3778 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3779 ( &__kmp_set_queuing_lock_flags );
3783 #if KMP_USE_ADAPTIVE_LOCKS
3785 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3786 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3788 __kmp_get_user_lock_owner_ =
3789 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3790 ( &__kmp_get_queuing_lock_owner );
3792 if ( __kmp_env_consistency_check ) {
3793 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3796 KMP_BIND_USER_LOCK(adaptive);
3799 __kmp_destroy_user_lock_ =
3800 ( void ( * )( kmp_user_lock_p ) )
3801 ( &__kmp_destroy_adaptive_lock );
3803 __kmp_is_user_lock_initialized_ =
3804 ( int ( * )( kmp_user_lock_p ) )
3805 ( &__kmp_is_queuing_lock_initialized );
3807 __kmp_get_user_lock_location_ =
3808 (
const ident_t * ( * )( kmp_user_lock_p ) )
3809 ( &__kmp_get_queuing_lock_location );
3811 __kmp_set_user_lock_location_ =
3812 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3813 ( &__kmp_set_queuing_lock_location );
3815 __kmp_get_user_lock_flags_ =
3816 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3817 ( &__kmp_get_queuing_lock_flags );
3819 __kmp_set_user_lock_flags_ =
3820 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3821 ( &__kmp_set_queuing_lock_flags );
3825 #endif // KMP_USE_ADAPTIVE_LOCKS
3828 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3829 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3831 __kmp_get_user_lock_owner_ =
3832 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3833 ( &__kmp_get_drdpa_lock_owner );
3835 if ( __kmp_env_consistency_check ) {
3836 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3837 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3840 KMP_BIND_USER_LOCK(drdpa);
3841 KMP_BIND_NESTED_USER_LOCK(drdpa);
3844 __kmp_destroy_user_lock_ =
3845 ( void ( * )( kmp_user_lock_p ) )
3846 ( &__kmp_destroy_drdpa_lock );
3848 __kmp_is_user_lock_initialized_ =
3849 ( int ( * )( kmp_user_lock_p ) )
3850 ( &__kmp_is_drdpa_lock_initialized );
3852 __kmp_get_user_lock_location_ =
3853 (
const ident_t * ( * )( kmp_user_lock_p ) )
3854 ( &__kmp_get_drdpa_lock_location );
3856 __kmp_set_user_lock_location_ =
3857 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3858 ( &__kmp_set_drdpa_lock_location );
3860 __kmp_get_user_lock_flags_ =
3861 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3862 ( &__kmp_get_drdpa_lock_flags );
3864 __kmp_set_user_lock_flags_ =
3865 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3866 ( &__kmp_set_drdpa_lock_flags );
3876 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3877 kmp_user_lock_p __kmp_lock_pool = NULL;
3880 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3881 int __kmp_num_locks_in_block = 1;
3883 static kmp_lock_index_t
3884 __kmp_lock_table_insert( kmp_user_lock_p lck )
3887 kmp_lock_index_t index;
3888 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3889 kmp_lock_index_t size;
3890 kmp_user_lock_p *table;
3893 if ( __kmp_user_lock_table.allocated == 0 ) {
3897 size = __kmp_user_lock_table.allocated * 2;
3899 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3900 KMP_MEMCPY( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3901 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3906 __kmp_user_lock_table.table = table;
3907 __kmp_user_lock_table.allocated = size;
3909 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3910 index = __kmp_user_lock_table.used;
3911 __kmp_user_lock_table.table[ index ] = lck;
3912 ++ __kmp_user_lock_table.used;
3916 static kmp_user_lock_p
3917 __kmp_lock_block_allocate()
3920 static int last_index = 0;
3921 if ( ( last_index >= __kmp_num_locks_in_block )
3922 || ( __kmp_lock_blocks == NULL ) ) {
3926 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3927 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3928 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
3930 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
3931 new_block->next_block = __kmp_lock_blocks;
3932 new_block->locks = (
void *)buffer;
3935 __kmp_lock_blocks = new_block;
3937 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
3938 [ last_index * __kmp_user_lock_size ] ) );
3948 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3949 kmp_lock_flags_t flags )
3951 kmp_user_lock_p lck;
3952 kmp_lock_index_t index;
3953 KMP_DEBUG_ASSERT( user_lock );
3955 __kmp_acquire_lock( &__kmp_global_lock, gtid );
3957 if ( __kmp_lock_pool == NULL ) {
3959 if ( __kmp_num_locks_in_block <= 1 ) {
3960 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
3963 lck = __kmp_lock_block_allocate();
3968 index = __kmp_lock_table_insert( lck );
3972 lck = __kmp_lock_pool;
3973 index = __kmp_lock_pool->pool.index;
3974 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3981 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3982 * ( (kmp_lock_index_t *) user_lock ) = index;
3985 * ( (kmp_user_lock_p *) user_lock ) = lck;
3989 __kmp_set_user_lock_flags( lck, flags );
3991 __kmp_release_lock( & __kmp_global_lock, gtid );
3998 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
4000 kmp_lock_pool_t * lock_pool;
4002 KMP_DEBUG_ASSERT( user_lock != NULL );
4003 KMP_DEBUG_ASSERT( lck != NULL );
4005 __kmp_acquire_lock( & __kmp_global_lock, gtid );
4007 lck->pool.next = __kmp_lock_pool;
4008 __kmp_lock_pool = lck;
4009 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4010 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
4011 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
4012 lck->pool.index = index;
4015 __kmp_release_lock( & __kmp_global_lock, gtid );
4019 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
4021 kmp_user_lock_p lck = NULL;
4023 if ( __kmp_env_consistency_check ) {
4024 if ( user_lock == NULL ) {
4025 KMP_FATAL( LockIsUninitialized, func );
4029 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4030 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
4031 if ( __kmp_env_consistency_check ) {
4032 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
4033 KMP_FATAL( LockIsUninitialized, func );
4036 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
4037 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4038 lck = __kmp_user_lock_table.table[index];
4041 lck = *( (kmp_user_lock_p *)user_lock );
4044 if ( __kmp_env_consistency_check ) {
4045 if ( lck == NULL ) {
4046 KMP_FATAL( LockIsUninitialized, func );
4054 __kmp_cleanup_user_locks(
void )
4061 __kmp_lock_pool = NULL;
4063 #define IS_CRITICAL(lck) \
4064 ( ( __kmp_get_user_lock_flags_ != NULL ) && \
4065 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) )
4095 while ( __kmp_user_lock_table.used > 1 ) {
4102 kmp_user_lock_p lck = __kmp_user_lock_table.table[
4103 --__kmp_user_lock_table.used ];
4105 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
4106 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
4113 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
4114 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
4116 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
4117 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.line );
4118 __kmp_str_loc_free( &str_loc);
4122 if ( IS_CRITICAL( lck ) ) {
4123 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
4126 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
4134 __kmp_destroy_user_lock( lck );
4140 if ( __kmp_lock_blocks == NULL ) {
4150 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4151 __kmp_user_lock_table.table = NULL;
4152 __kmp_user_lock_table.allocated = 0;
4154 while ( table_ptr != NULL ) {
4159 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
4160 __kmp_free( table_ptr );
4167 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4168 __kmp_lock_blocks = NULL;
4170 while ( block_ptr != NULL ) {
4171 kmp_block_of_locks_t *next = block_ptr->next_block;
4172 __kmp_free( block_ptr->locks );
4179 TCW_4(__kmp_init_user_locks, FALSE);
4182 #endif // KMP_USE_DYNAMIC_LOCK