[Date Prev][Date Next]
[Chronological]
[Thread]
[Top]
Re: (ITS#4640) slapd lockup
--On Thursday, August 17, 2006 5:54 PM +0000 quanah@stanford.edu wrote:
>
>
> --On Thursday, August 17, 2006 6:21 AM -0700 Quanah Gibson-Mount
> <quanah@stanford.edu> wrote:
>
>>
>>
>> --On Wednesday, August 16, 2006 7:29 PM -0700 Quanah Gibson-Mount
>> <quanah@stanford.edu> wrote:
>>
>>>
>>>
>>> --On Wednesday, August 16, 2006 8:33 AM +0000 openldap-its@OpenLDAP.org
>>> wrote:
>>>
>>>
>>> This has happened again, and now I'm seeing slapd grow at crazy rates
>>> even though I stopped incoming connections to it by dropping it out of
>>> the load balance pool:
>>
>> I applied Howard's patches from HEAD to daemon.c and connection.c, and
>> then deployed that build to ldap4. Once again, the system went haywire,
>> but this time something interesting was logged:
>
> Okay, after this crash, I had another crash that generated a core dump.
> Here is the info from it:
>
> Core was generated by `/usr/local/lib/slapd -h ldap:///'.
> Program terminated with signal 6, Aborted.
Relevant code snippets:
> Thread 10 (process 16773):
># 0 0x00002b3109f38e2c in pthread_join () from /lib/libpthread.so.0
># 1 0x0000000000425b21 in slapd_daemon () at daemon.c:2242
(gdb) l
2237 "listener ldap_pvt_thread_create failed
(%d)\n", rc, 0, 0 );
2238 return rc;
2239 }
2240
2241 /* wait for the listener thread to complete */
2242 ldap_pvt_thread_join( listener_tid, (void *) NULL );
2243 }
2244 #else
2245 /* experimental code */
2246 slapd_daemon_task( NULL );
># 2 0x00000000004160d4 in main (argc=3, argv=0x7fffff99fe38) at main.c:854
(gdb) l
849 if (is_NT_Service)
850 lutil_LogStartedEvent( serverName, slap_debug, configfile ?
851 configfile : SLAPD_DEFAULT_CONFIGFILE , urls );
852 #endif
853
854 rc = slapd_daemon();
855
856 #ifdef HAVE_NT_SERVICE_MANAGER
857 /* Throw away the event that we used during the startup
process. */
858 if ( is_NT_Service )
> Thread 9 (process 16774):
># 0 0x00002b310a119b2c in epoll_wait () from /lib/libc.so.6
># 1 0x00000000004251b9 in slapd_daemon_task (ptr=0x6) at daemon.c:1843
(gdb) l
1838 Debug( LDAP_DEBUG_CONNS,
1839 "daemon: select: listen=%d
active_threads=%d tvp=%s\n",
1840 lr->sl_sd, at, tvp == NULL ? "NULL"
: "zero" );
1841 }
1842
1843 switch(ns = SLAP_EVENT_WAIT(tvp)) {
1844 case -1: { /* failure - try again */
1845 int err = sock_errno();
1846
1847 if( err != EINTR ) {
># 148 0x0000000000424ab0 in slap_listener_thread () at daemon.c:1422
(gdb) l
1417
"cn=peercred,cn=external,cn=auth" ) + 1 );
1418 authid.bv_len = sprintf(
authid.bv_val,
1419 "gidNumber=%d+uidNumber=%d,"
1420
"cn=peercred,cn=external,cn=auth",
1421 (int) gid, (int) uid );
1422 assert( authid.bv_len <=
1423 STRLENOF(
"gidNumber=4294967295+uidNumber=4294967295,"
1424
"cn=peercred,cn=external,cn=auth" ) );
1425 }
1426 }
> (gdb) thread 8
> [Switching to thread 8 (process 16775)]#0 0x00002b3109f3a4e4 in
> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
> (gdb) bt
># 0 0x00002b3109f3a4e4 in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib/libpthread.so.0
># 1 0x00002b31093f25ae in ldap_int_thread_pool_wrapper (xpool=0x6361b0)
># at
> tpool.c:490
485
486 if (pool->ltp_state ==
LDAP_INT_THREAD_POOL_PAUSING) {
487 if (pool->ltp_active_count < 2) {
488
ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
489 }
490 ldap_pvt_thread_cond_wait(&pool->ltp_cond,
&pool->ltp_mutex);
491 }
492 }
493
494 for ( i=0; i<MAXKEYS && ltc_key[i].ltk_key; i++ ) {
># 148 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
(gdb) l
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> (gdb) thread 7
> [Switching to thread 7 (process 16776)]#0 0x00002b3109f3a4e4 in
> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
> (gdb) bt
># 0 0x00002b3109f3a4e4 in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib/libpthread.so.0
># 1 0x00002b31093f25ae in ldap_int_thread_pool_wrapper (xpool=0x6361b0)
># at
> tpool.c:490
(gdb) l
485
486 if (pool->ltp_state ==
LDAP_INT_THREAD_POOL_PAUSING) {
487 if (pool->ltp_active_count < 2) {
488
ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
489 }
490 ldap_pvt_thread_cond_wait(&pool->ltp_cond,
&pool->ltp_mutex);
491 }
492 }
493
494 for ( i=0; i<MAXKEYS && ltc_key[i].ltk_key; i++ ) {
># 148 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> (gdb) thread 6
> [Switching to thread 6 (process 16810)]#0 0x00002b3109f3a4e4 in
> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
> (gdb) bt
># 0 0x00002b3109f3a4e4 in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib/libpthread.so.0
># 1 0x00002b31093f25ae in ldap_int_thread_pool_wrapper (xpool=0x6361b0)
># at
> tpool.c:490
(gdb) l
485
486 if (pool->ltp_state ==
LDAP_INT_THREAD_POOL_PAUSING) {
487 if (pool->ltp_active_count < 2) {
488
ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
489 }
490 ldap_pvt_thread_cond_wait(&pool->ltp_cond,
&pool->ltp_mutex);
491 }
492 }
493
494 for ( i=0; i<MAXKEYS && ltc_key[i].ltk_key; i++ ) {
># 148 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> (gdb) thread 5
> [Switching to thread 5 (process 16826)]#0 0x00002b3109f3a4e4 in
> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
> (gdb) bt
># 0 0x00002b3109f3a4e4 in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib/libpthread.so.0
># 1 0x00002b31093f25ae in ldap_int_thread_pool_wrapper (xpool=0x6361b0)
># at
> tpool.c:490
485
486 if (pool->ltp_state ==
LDAP_INT_THREAD_POOL_PAUSING) {
487 if (pool->ltp_active_count < 2) {
488
ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
489 }
490 ldap_pvt_thread_cond_wait(&pool->ltp_cond,
&pool->ltp_mutex);
491 }
492 }
493
494 for ( i=0; i<MAXKEYS && ltc_key[i].ltk_key; i++ ) {
># 148 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> [Switching to thread 4 (process 16867)]#0 0x00002b310a0c7028 in memcpy
> () from /lib/libc.so.6
> (gdb) bt
># 0 0x00002b310a0c7028 in memcpy () from /lib/libc.so.6
># 1 0x00002b310a0c6629 in memmove () from /lib/libc.so.6
># 2 0x00002b310b19e5ca in hdb_idl_cache_get (bdb=0x71d960,
> db=0x2aaaac6c37a0, key=0x1c94a, ids=0x436fe250) at idl.c:319
314 ldap_pvt_thread_rdwr_rlock( &bdb->bi_idl_tree_rwlock );
315 matched_idl_entry = avl_find( bdb->bi_idl_tree, &idl_tmp,
316 bdb_idl_entry_cmp );
317 if ( matched_idl_entry != NULL ) {
318 if ( matched_idl_entry->idl && ids )
319 BDB_IDL_CPY( ids, matched_idl_entry->idl );
320 ldap_pvt_thread_mutex_lock(
&bdb->bi_idl_tree_lrulock );
321 IDL_LRU_DELETE( bdb, matched_idl_entry );
322 IDL_LRU_ADD( bdb, matched_idl_entry );
323 ldap_pvt_thread_mutex_unlock(
&bdb->bi_idl_tree_lrulock );
># 3 0x00002b310b19ce81 in hdb_dn2idl_internal (cx=0x4367dd50) at
># dn2id.c:877
872 cx->key.data = ptr;
873 cx->key.size = sizeof(ID)+1;
874 if ( cx->prefix == DN_SUBTREE_PREFIX ) {
875 ID *ids = cx->depth ? cx->tmp : cx->ids;
876 *ptr = cx->prefix;
877 cx->rc = bdb_idl_cache_get(cx->bdb, cx->db,
&cx->key, ids);
878 if ( cx->rc == LDAP_SUCCESS ) {
879 if ( cx->depth ) {
880 bdb_idl_append( cx->ids,
cx->tmp );
881 cx->need_sort = 1;
> #4 0x00002b310b19d484 in hdb_dn2idl (op=0x2aaab2d02e10,
># e=0x4367e200,
> ids=0x436fe250, stack=0x2aaab1200010) at dn2id.c:1112
1107 cx.key.size = sizeof(ID);
1108 cx.key.flags = DB_DBT_USERMEM;
1109
1110 DBTzero(&cx.data);
1111
1112 hdb_dn2idl_internal(&cx);
1113 if ( cx.need_sort ) {
1114 char *ptr = ((char *)&cx.id)-1;
1115 if ( !BDB_IDL_IS_RANGE( cx.ids ) && cx.ids[0] > 3 )
1116 bdb_idl_sort( cx.ids, cx.tmp );
># 5 0x00002b310b196496 in hdb_search (op=0x2aaab2d02e10, rs=0x437ff370)
># at
> search.c:1119
1114 }
1115
1116 if( op->ors_deref & LDAP_DEREF_SEARCHING ) {
1117 rc = search_aliases( op, rs, e, locker, ids,
scopes, stack );
1118 } else {
1119 rc = bdb_dn2idl( op, e, ids, stack );
1120 }
1121
1122 if ( rc == LDAP_SUCCESS ) {
1123 rc = bdb_filter_candidates( op, &f, ids,
># 6 0x000000000042a0ca in fe_op_search (op=0x2aaab2d02e10, rs=0x437ff370)
> at search.c:355
350 }
351
352 /* actually do the search and send the result(s) */
353 if ( op->o_bd->be_search ) {
354 if ( limits_check( op, rs ) == 0 ) {
355 (op->o_bd->be_search)( op, rs );
356 }
357 /* else limits_check() sends error */
358
359 } else {
># 7 0x0000000000429a05 in do_search (op=0x2aaab2d02e10, rs=0x437ff370) at
> search.c:217
212 op->o_log_prefix, abuf, 0, 0, 0 );
213 }
214 }
215
216 op->o_bd = frontendDB;
217 rs->sr_err = frontendDB->be_search( op, rs );
218
219 return_results:;
220 if ( !BER_BVISNULL( &op->o_req_dn ) ) {
221 slap_sl_free( op->o_req_dn.bv_val, op->o_tmpmemctx
);
># 8 0x000000000042813d in connection_operation (ctx=0x437ff4e0,
> arg_v=0x2aaab2d02e10) at connection.c:1090
1085 assert( 0 );
1086 }
1087
1088 assert( opidx > -1 );
1089 INCR_OP_INITIATED( opidx );
1090 rc = (*(opfun[opidx]))( op, &rs );
1091
1092 operations_error:
1093 if ( rc == SLAPD_DISCONNECT ) {
1094 tag = LBER_ERROR;
># 9 0x00000000004287f3 in connection_read_thread (ctx=0x437ff4e0,
># argv=0x0)
> at connection.c:1219
1214 return (void*)(long)rc;
1215 }
1216
1217 /* execute a single queued request in the same thread */
1218 if( cri.op && !cri.nullop ) {
1219 rc = (long)connection_operation( ctx, cri.op );
1220 } else if ( cri.func ) {
1221 rc = (long)cri.func( ctx, cri.arg );
1222 }
1223
># 10 0x00002b31093f2522 in ldap_int_thread_pool_wrapper (xpool=0x437269d8)
> at tpool.c:478
473
474 LDAP_SLIST_INSERT_HEAD(&pool->ltp_active_list, ctx,
ltc_next.al);
475 pool->ltp_active_count++;
476 ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
477
478 ctx->ltc_start_routine(ltc_key, ctx->ltc_arg);
479
480 ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
481 LDAP_SLIST_REMOVE(&pool->ltp_active_list, ctx,
482 ldap_int_thread_ctx_s, ltc_next.al);
># 157 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> [Switching to thread 3 (process 16905)]#0 0x00002b3109f3a4e4 in
> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
> (gdb) bt
># 0 0x00002b3109f3a4e4 in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib/libpthread.so.0
># 1 0x00002b31093f25ae in ldap_int_thread_pool_wrapper (xpool=0x6361b0)
># at
> tpool.c:490
485
486 if (pool->ltp_state ==
LDAP_INT_THREAD_POOL_PAUSING) {
487 if (pool->ltp_active_count < 2) {
488
ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
489 }
490 ldap_pvt_thread_cond_wait(&pool->ltp_cond,
&pool->ltp_mutex);
491 }
492 }
493
494 for ( i=0; i<MAXKEYS && ltc_key[i].ltk_key; i++ ) {
># 148 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> [Switching to thread 2 (process 16911)]#0 0x00002b3109f3a4e4 in
> pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
> (gdb) bt
># 0 0x00002b3109f3a4e4 in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib/libpthread.so.0
># 1 0x00002b31093f25ae in ldap_int_thread_pool_wrapper (xpool=0x6361b0)
># at
> tpool.c:490
485
486 if (pool->ltp_state ==
LDAP_INT_THREAD_POOL_PAUSING) {
487 if (pool->ltp_active_count < 2) {
488
ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
489 }
490 ldap_pvt_thread_cond_wait(&pool->ltp_cond,
&pool->ltp_mutex);
491 }
492 }
493
494 for ( i=0; i<MAXKEYS && ltc_key[i].ltk_key; i++ ) {
># 148 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
> [Switching to thread 1 (process 16827)]#0 0x00002b310a076545 in raise ()
> from /lib/libc.so.6
> (gdb) bt
># 0 0x00002b310a076545 in raise () from /lib/libc.so.6
># 1 0x00002b310a077cce in abort () from /lib/libc.so.6
># 2 0x00002b310a070362 in __assert_fail () from /lib/libc.so.6
># 3 0x00000000004269bd in connection_init (s=196, listener=0x603100,
> dnsname=0x42fff000 "tree2.stanford.edu", peername=0x42ffdff0
> "IP=171.64.15.236:56444", flags=0, ssf=0, authid=0x0) at connection.c:505
500 assert( c->c_sasl_authctx == NULL );
501 assert( c->c_sasl_sockctx == NULL );
502 assert( c->c_sasl_extra == NULL );
503 assert( c->c_sasl_bindop == NULL );
504 assert( c->c_currentber == NULL );
505 assert( c->c_writewaiter == 0);
506
507 c->c_listener = listener;
508
509 if ( flags == CONN_IS_CLIENT ) {
># 4 0x0000000000424568 in slap_listener_thread (ctx=0x4185, ptr=0x41bb)
># at
> daemon.c:1506
1501 }
1502 }
1503 #endif /* HAVE_TCPD */
1504 }
1505
1506 id = connection_init(s, sl,
1507 dnsname != NULL ? dnsname : SLAP_STRING_UNKNOWN,
1508 peername,
1509 #ifdef HAVE_TLS
1510 sl->sl_is_tls ? CONN_IS_TLS : 0,
># 5 0x00002b31093f2522 in ldap_int_thread_pool_wrapper (xpool=0x4185) at
> tpool.c:478
473
474 LDAP_SLIST_INSERT_HEAD(&pool->ltp_active_list, ctx,
ltc_next.al);
475 pool->ltp_active_count++;
476 ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
477
478 ctx->ltc_start_routine(ltc_key, ctx->ltc_arg);
479
480 ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
481 LDAP_SLIST_REMOVE(&pool->ltp_active_list, ctx,
482 ldap_int_thread_ctx_s, ltc_next.al);
># 152 0x00002b31093f2400 in ldap_int_thread_pool_shutdown () at tpool.c:118
113 while ((pool =
LDAP_STAILQ_FIRST(&ldap_int_thread_pool_list)) != NULL) {
114 (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore
thr_debug macro */
115 }
116 ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
117 return(0);
118 }
119
120 int
121 ldap_pvt_thread_pool_init (
122 ldap_pvt_thread_pool_t *tpool,
--Quanah
--
Quanah Gibson-Mount
Principal Software Developer
ITS/Shared Application Services
Stanford University
GnuPG Public Key: http://www.stanford.edu/~quanah/pgp.html