[Date Prev][Date Next] [Chronological] [Thread] [Top]

hangs of test036 and so



I have a clean stack backtrace of a hang of test036:

(gdb) thread apply all bt

Thread 18 (Thread 1082132832 (LWP 7326)):
#0  0x00000031ce4bebe6 in __select_nocancel () from /lib64/tls/libc.so.6
#1  0x00000000004447ba in slapd_daemon_task (ptr=0x0) at daemon.c:1844
#2  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#3  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#4  0x0000000000000000 in ?? ()

Thread 17 (Thread 1090525536 (LWP 7328)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 16 (Thread 1098918240 (LWP 7342)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 15 (Thread 1107310944 (LWP 7347)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 14 (Thread 1115703648 (LWP 7355)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 13 (Thread 1124096352 (LWP 7356)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 12 (Thread 1132489056 (LWP 7357)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 11 (Thread 1140881760 (LWP 7358)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 10 (Thread 1149274464 (LWP 7369)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 9 (Thread 1157667168 (LWP 7370)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 8 (Thread 1166059872 (LWP 7371)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 7 (Thread 1174452576 (LWP 7372)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 6 (Thread 1182845280 (LWP 7373)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 5 (Thread 1191237984 (LWP 7374)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 4 (Thread 1199630688 (LWP 7378)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 3 (Thread 1208023392 (LWP 7379)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 2 (Thread 1216416096 (LWP 7380)):
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 1 (Thread 182895213568 (LWP 7324)):
#0  0x00000031cf306f2b in pthread_join () from /lib64/tls/libpthread.so.0
#1  0x00000000005c2858 in ldap_pvt_thread_join (thread=1082132832, 
    thread_return=0x0) at thr_posix.c:186
#2  0x000000000044545f in slapd_daemon () at daemon.c:2215
#3  0x000000000042fb0d in main (argc=8, argv=0x7fbffff178) at main.c:785

So apparently all threads are waiting on the very same condition; e.g.
thread #2:

(gdb) thread 2
[Switching to thread 2 (Thread 1216416096 (LWP 7380))]#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/tls/libpthread.so.0
(gdb) bt full
#0  0x00000031cf3088da in pthread_cond_wait@@GLIBC_2.3.2 ()
   from /lib64/tls/libpthread.so.0
No symbol table info available.
#1  0x00000000005c2900 in ldap_pvt_thread_cond_wait (cond=0x8c8f10, 
    mutex=0x8c8ee8) at thr_posix.c:282
No locals.
#2  0x00000000005c16a6 in ldap_int_thread_pool_wrapper (xpool=0x8c8ee0)
    at tpool.c:603
        pool = (struct ldap_int_thread_pool_s *) 0x8c8ee0
        ctx = (ldap_int_thread_ctx_t *) 0x0
        ltc_key = {{ltk_key = 0x4a4476, ltk_data = 0xa876a0, 
    ltk_free = 0x4a4260 <slap_sl_mem_destroy>}, {ltk_key = 0x8a23c0, 
    ltk_data = 0xa876e0, ltk_free = 0x55301b <meta_back_candidates_keyfree>}, {
    ltk_key = 0x0, ltk_data = 0x0, ltk_free = 0} <repeats 30 times>}
        tid = 1216416096
        i = 306
        keyslot = 306
        hash = 306
#3  0x00000031cf3060aa in start_thread () from /lib64/tls/libpthread.so.0
No symbol table info available.
#4  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
No symbol table info available.
#5  0x0000000000000000 in ?? ()
No symbol table info available.
(gdb) p ((struct ldap_int_thread_pool_s*)0x8c8ee0)[0]
$1 = {ltp_next = {stqe_next = 0x0}, ltp_mutex = {__m_reserved = 0, 
    __m_count = 0, __m_owner = 0x1000000000, __m_kind = 0, __m_lock = {
      __status = 0, __spinlock = 0}}, ltp_cond = {__c_lock = {
      __status = 26345329393664, __spinlock = 3075}, __c_waiting = 0xbf3, 
    __padding = "ïv\000\000\000\000\000\000ï216\214\000\000\000\000", 
    __align = 32}, ltp_pcond = {__c_lock = {__status = 0, __spinlock = 0}, 
    __c_waiting = 0x0, __padding = '\0' <repeats 15 times>, __align = 0}, 
  ltp_pending_list = {stqh_first = 0x0, stqh_last = 0x8c8f70}, 
  ltp_free_list = {slh_first = 0x9c7930}, ltp_active_list = {slh_first = 0x0}, 
  ltp_state = LDAP_INT_THREAD_POOL_RUNNING, ltp_max_count = 16, 
  ltp_max_pending = 0, ltp_pending_count = 0, ltp_active_count = 0, 
  ltp_open_count = 16, ltp_starting = 0}

except thread #18 which is waiting on a select():

[Switching to thread 18 (Thread 1082132832 (LWP 7326))]#0
0x00000031ce4bebe6 in __select_nocancel () from /lib64/tls/libc.so.6
(gdb) bt full
#0  0x00000031ce4bebe6 in __select_nocancel () from /lib64/tls/libc.so.6
No symbol table info available.
#1  0x00000000004447ba in slapd_daemon_task (ptr=0x0) at daemon.c:1844
        i = 50
        nrfds = 0
        tdelta = 1
        ns = 1
        now = 1132167331
        tvp = (struct timeval *) 0x0
        cat = (struct timeval *) 0x0
        nwriters = 0
        readfds = {__fds_bits = {324261991214710864, 0 <repeats 63
times>}}
        writefds = {__fds_bits = {0 <repeats 64 times>}}
        tv = {tv_sec = 0, tv_usec = 0}
        rtask = (struct re_s *) 0x0
        at = 0
        nfds = 59
        nwfds = 0
        l = 1
        last_idle_check = 0
        idle = {tv_sec = 0, tv_usec = 0}
        ebadf = 0
#2  0x00000031cf3060aa in start_thread ()
from /lib64/tls/libpthread.so.0
No symbol table info available.
#3  0x00000031ce4c5b43 in clone () from /lib64/tls/libc.so.6
No symbol table info available.
#4  0x0000000000000000 in ?? ()
No symbol table info available.

Anything specific I should be looking at?

p.




Ing. Pierangelo Masarati
Responsabile Open Solution

SysNet s.n.c.
Via Dossi, 8 - 27100 Pavia - ITALIA
http://www.sys-net.it
------------------------------------------
Office:   +39.02.23998309          
Mobile:   +39.333.4963172
Email:    pierangelo.masarati@sys-net.it
------------------------------------------