[Date Prev][Date Next] [Chronological] [Thread] [Top]

(ITS#6308) replication locks with delta-syncrepl



Full_Name: Quanah Gibson-Mount
Version: 2.4.18
OS: 
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (75.111.29.239)


OpenLDAP 2.4.18 with BDB 4.7.25, fully patched.

Delta-sync replication works fine as long as the system is not loaded.  Once it
is loaded, replication will fail after a while, waiting on a transaction.

>From the log with sync logging:

Sep 28 11:08:14 ldap1 slapd[3241]: do_syncrep2:
cookie=csn=20090928180814Z#000000#00#000000,rid=000
Sep 28 11:08:14 ldap1 slapd[3241]: syncrepl_message_to_op: rid=000 be_modify
suRegID=109cd894e76211d1bcdf2436000baa77,cn=People,dc=Stanford,dc=edu (0)
Sep 28 11:09:18 ldap1 slapd[3241]: do_syncrep2:
cookie=csn=20090928180918Z#000000#00#000000,rid=000


Backtrace shows:

Thread 10 (Thread 0x40891950 (LWP 3243)):
#0  0x00007f05b99e5b78 in epoll_wait () from /lib/libc.so.6
#1  0x00000000004392e6 in slapd_daemon_task (ptr=0x0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/daemon.c:2460
#2  0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#3  0x00007f05b99e55ad in clone () from /lib/libc.so.6
#4  0x0000000000000000 in ?? ()

Thread 9 (Thread 0x4278c950 (LWP 3244)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb96650b in ldap_pvt_thread_cond_wait (cond=0x2210980,
mutex=0x2210958) at /tmp/buildd/openldap-2.4.18/libraries/libldap_r/thr_posix.c:277
#2  0x00007f05bb964f6f in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:672
#3  0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#4  0x00007f05b99e55ad in clone () from /lib/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 8 (Thread 0x42f8d950 (LWP 3245)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb96650b in ldap_pvt_thread_cond_wait (cond=0x2210980,
mutex=0x2210958) at /tmp/buildd/openldap-2.4.18/libraries/libldap_r/thr_posix.c:277
#2  0x00007f05bb964f6f in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:672
#3  0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#4  0x00007f05b99e55ad in clone () from /lib/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 7 (Thread 0x4378e950 (LWP 3246)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb96650b in ldap_pvt_thread_cond_wait (cond=0x2210980,
mutex=0x2210958) at /tmp/buildd/openldap-2.4.18/libraries/libldap_r/thr_posix.c:277
#2  0x00007f05bb964f6f in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:672
#3  0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#4  0x00007f05b99e55ad in clone () from /lib/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 6 (Thread 0x41466950 (LWP 3247)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb38154a in __db_pthread_mutex_lock (env=0x23df7d0, mutex=1575681)
at ../dist/../mutex/mut_pthread.c:224
#2  0x00007f05bb43adc2 in __lock_get_internal (lt=0x23df910,
sh_locker=0x7f042992e118, flags=0, obj=0x7f03e98ebfa0, lock_mode=DB_LOCK_READ,
timeout=0, lock=0x41263868)
    at ../dist/../lock/lock.c:946
#3  0x00007f05bb437f9d in __lock_vec (env=0x23df7d0, sh_locker=0x7f042992e118,
flags=0, list=0x41263850, nlist=2, elistp=0x41263848) at
../dist/../lock/lock.c:136
#4  0x00007f05bb492ff6 in __db_lget (dbc=0x7f03e98ebec0, action=2, pgno=2766,
mode=DB_LOCK_READ, lkflags=0, lockp=0x41263ac0) at ../dist/../db/db_meta.c:1062
#5  0x00007f05bb39fe1c in __bam_search (dbc=0x7f03e98ebec0, root_pgno=1,
key=0x412e4140, flags=1409, slevel=1, recnop=0x0, exactp=0x41263e24) at
../dist/../btree/bt_search.c:499
#6  0x00007f05bb38ac89 in __bamc_search (dbc=0x7f03e98ebec0, root_pgno=0,
key=0x412e4140, flags=26, exactp=0x41263e24) at
../dist/../btree/bt_cursor.c:2501
#7  0x00007f05bb386233 in __bamc_get (dbc=0x7f03e98ebec0, key=0x412e4140,
data=0x412e4050, flags=26, pgnop=0x41263ed4) at
../dist/../btree/bt_cursor.c:970
#8  0x00007f05bb47d4bb in __dbc_get (dbc_arg=0x76697e0, key=0x412e4140,
data=0x412e4050, flags=26) at ../dist/../db/db_cam.c:700
#9  0x00007f05bb48dfeb in __dbc_get_pp (dbc=0x76697e0, key=0x412e4140,
data=0x412e4050, flags=8218) at ../dist/../db/db_iface.c:2095
#10 0x00007f05b6e33d6d in hdb_idl_fetch_key (be=0x41464990, db=0x7271e30,
txn=0x2bb4270, key=0x412e4140, ids=0x7f0404d76010, saved_cursor=0x0, get_flag=0)
at idl.c:605
#11 0x00007f05b6e29e91 in hdb_key_read (be=0x41464990, db=0x7271e30,
txn=0x2bb4270, k=0x2bb53c8, ids=0x7f0404d76010, saved_cursor=0x0, get_flag=0) at
key.c:50
#12 0x00007f05b6e2c6c1 in equality_candidates (op=0xc9bac80, rtxn=0x2bb4270,
ava=0x2bb5190, ids=0x7f0404f76010, tmp=0x7f0404d76010) at filterindex.c:788
#13 0x00007f05b6e2adb3 in hdb_filter_candidates (op=0xc9bac80, rtxn=0x2bb4270,
f=0x2bb51d8, ids=0x7f0404f76010, tmp=0x7f0404d76010, stack=0x7f0405076010) at
filterindex.c:154
#14 0x00007f05b6e2ba13 in list_candidates (op=0xc9bac80, rtxn=0x2bb4270,
flist=0x412e4610, ftype=161, ids=0x7f0404e76010, tmp=0x7f0404d76010,
save=0x7f0404f76010) at filterindex.c:581
#15 0x00007f05b6e2b33f in hdb_filter_candidates (op=0xc9bac80, rtxn=0x2bb4270,
f=0x412e45f0, ids=0x7f0404e76010, tmp=0x7f0404d76010, stack=0x7f0404f76010) at
filterindex.c:204
#16 0x00007f05b6e2ba13 in list_candidates (op=0xc9bac80, rtxn=0x2bb4270,
flist=0x412e45d0, ftype=160, ids=0x41364820, tmp=0x7f0404d76010,
save=0x7f0404e76010) at filterindex.c:581
#17 0x00007f05b6e2b287 in hdb_filter_candidates (op=0xc9bac80, rtxn=0x2bb4270,
f=0x412e4630, ids=0x41364820, tmp=0x7f0404d76010, stack=0x7f0404e76010) at
filterindex.c:198
#18 0x00007f05b6e26e19 in search_candidates (op=0xc9bac80, rs=0x41465ca0,
e=0x412e47d0, txn=0x2bb4270, ids=0x41364820, scopes=0x412e4820) at
search.c:1206
#19 0x00007f05b6e25090 in hdb_search (op=0xc9bac80, rs=0x41465ca0) at
search.c:586
#20 0x00000000004c73c6 in overlay_op_walk (op=0xc9bac80, rs=0x41465ca0,
which=op_search, oi=0x22a0c90, on=0x0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:669
#21 0x00000000004c75ed in over_op_func (op=0xc9bac80, rs=0x41465ca0,
which=op_search) at /tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:721
#22 0x00000000004c76bb in over_op_search (op=0xc9bac80, rs=0x41465ca0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:748
#23 0x0000000000440b4c in fe_op_search (op=0xc9bac80, rs=0x41465ca0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/search.c:366
#24 0x00000000004404b7 in do_search (op=0xc9bac80, rs=0x41465ca0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/search.c:217
#25 0x000000000043cede in connection_operation (ctx=0x41465df0, arg_v=0xc9bac80)
at /tmp/buildd/openldap-2.4.18/servers/slapd/connection.c:1123
#26 0x000000000043d46a in connection_read_thread (ctx=0x41465df0, argv=0x1cb) at
/tmp/buildd/openldap-2.4.18/servers/slapd/connection.c:1259
#27 0x00007f05bb96500d in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:685
#28 0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#29 0x00007f05b99e55ad in clone () from /lib/libc.so.6
#30 0x0000000000000000 in ?? ()

Thread 5 (Thread 0x41c67950 (LWP 3248)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb38154a in __db_pthread_mutex_lock (env=0x23df7d0, mutex=1575506)
at ../dist/../mutex/mut_pthread.c:224
#2  0x00007f05bb43adc2 in __lock_get_internal (lt=0x23df910,
sh_locker=0x7f042992dad8, flags=0, obj=0x41ae5510, lock_mode=DB_LOCK_READ,
timeout=0, lock=0x41ae5790)
    at ../dist/../lock/lock.c:946
#3  0x00007f05bb438efc in __lock_get_api (env=0x23df7d0, locker=2147483856,
flags=0, obj=0x41ae5510, lock_mode=DB_LOCK_READ, lock=0x41ae5790) at
../dist/../lock/lock.c:423
#4  0x00007f05bb438d8c in __lock_get_pp (dbenv=0x23e5360, locker=2147483856,
flags=0, obj=0x41ae5510, lock_mode=DB_LOCK_READ, lock=0x41ae5790) at
../dist/../lock/lock.c:395
#5  0x00007f05b6e3731b in bdb_cache_entry_db_lock (bdb=0x229dcc0, txn=0x2d3df60,
ei=0x7f03fae33d20, rw=0, tryOnly=0, lock=0x41ae5790) at cache.c:227
#6  0x00007f05b6e389bc in hdb_cache_find_id (op=0x7f03c12c1bd0, tid=0x2d3df60,
id=294299, eip=0x41ae57c0, flag=0, lock=0x41ae5790) at cache.c:979
#7  0x00007f05b6e256a8 in hdb_search (op=0x7f03c12c1bd0, rs=0x41c66ca0) at
search.c:707
#8  0x00000000004c73c6 in overlay_op_walk (op=0x7f03c12c1bd0, rs=0x41c66ca0,
which=op_search, oi=0x22a0c90, on=0x0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:669
#9  0x00000000004c75ed in over_op_func (op=0x7f03c12c1bd0, rs=0x41c66ca0,
which=op_search) at /tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:721
#10 0x00000000004c76bb in over_op_search (op=0x7f03c12c1bd0, rs=0x41c66ca0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:748
#11 0x0000000000440b4c in fe_op_search (op=0x7f03c12c1bd0, rs=0x41c66ca0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/search.c:366
#12 0x00000000004404b7 in do_search (op=0x7f03c12c1bd0, rs=0x41c66ca0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/search.c:217
#13 0x000000000043cede in connection_operation (ctx=0x41c66df0,
arg_v=0x7f03c12c1bd0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/connection.c:1123
#14 0x000000000043d46a in connection_read_thread (ctx=0x41c66df0, argv=0x13d) at
/tmp/buildd/openldap-2.4.18/servers/slapd/connection.c:1259
#15 0x00007f05bb96500d in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:685
#16 0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#17 0x00007f05b99e55ad in clone () from /lib/libc.so.6
#18 0x0000000000000000 in ?? ()

Thread 4 (Thread 0x43f8f950 (LWP 3249)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb96650b in ldap_pvt_thread_cond_wait (cond=0x2210980,
mutex=0x2210958) at /tmp/buildd/openldap-2.4.18/libraries/libldap_r/thr_posix.c:277
#2  0x00007f05bb964f6f in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:672
#3  0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#4  0x00007f05b99e55ad in clone () from /lib/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 3 (Thread 0x44790950 (LWP 3250)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb38154a in __db_pthread_mutex_lock (env=0x23df7d0, mutex=1574526)
at ../dist/../mutex/mut_pthread.c:224
#2  0x00007f05bb43adc2 in __lock_get_internal (lt=0x23df910,
sh_locker=0x7f042991e330, flags=0, obj=0x4478e960, lock_mode=DB_LOCK_WRITE,
timeout=0, lock=0x4478e948)
    at ../dist/../lock/lock.c:946
#3  0x00007f05bb437f9d in __lock_vec (env=0x23df7d0, sh_locker=0x7f042991e330,
flags=0, list=0x4478e900, nlist=2, elistp=0x0) at ../dist/../lock/lock.c:136
#4  0x00007f05bb437d7f in __lock_vec_api (env=0x23df7d0, lid=2147484554,
flags=0, list=0x4478e900, nlist=2, elistp=0x0) at ../dist/../lock/lock.c:84
#5  0x00007f05bb437ccd in __lock_vec_pp (dbenv=0x23e5360, lid=2147484554,
flags=0, list=0x4478e900, nlist=2, elistp=0x0) at ../dist/../lock/lock.c:66
#6  0x00007f05b6e37192 in hdb_cache_entry_db_relock (bdb=0x229dcc0,
txn=0x969d990, ei=0x7f03fae33d20, rw=1, tryOnly=0, lock=0x4478eae0) at
cache.c:192
#7  0x00007f05b6e392ce in hdb_cache_modify (bdb=0x229dcc0, e=0x7f0426b06978,
newAttrs=0x7f04169f0d98, txn=0x969d990, lock=0x4478eae0) at cache.c:1210
#8  0x00007f05b6e2027b in hdb_modify (op=0x4478f6a0, rs=0x4478f230) at
modify.c:663
#9  0x00000000004c73c6 in overlay_op_walk (op=0x4478f6a0, rs=0x4478f230,
which=op_modify, oi=0x22a0c90, on=0x0) at
/tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:669
#10 0x00000000004c75ed in over_op_func (op=0x4478f6a0, rs=0x4478f230,
which=op_modify) at /tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:721
#11 0x00000000004c7703 in over_op_modify (op=0x4478f6a0, rs=0x4478f230) at
/tmp/buildd/openldap-2.4.18/servers/slapd/backover.c:760
#12 0x00000000004b7a58 in syncrepl_message_to_op (si=0x22a1550, op=0x4478f6a0,
msg=0x9cf5c90) at /tmp/buildd/openldap-2.4.18/servers/slapd/syncrepl.c:1734
#13 0x00000000004b48ff in do_syncrep2 (op=0x4478f6a0, si=0x22a1550) at
/tmp/buildd/openldap-2.4.18/servers/slapd/syncrepl.c:873
#14 0x00000000004b6480 in do_syncrepl (ctx=0x4478fdf0, arg=0x229f730) at
/tmp/buildd/openldap-2.4.18/servers/slapd/syncrepl.c:1358
#15 0x000000000043d486 in connection_read_thread (ctx=0x4478fdf0, argv=0xd) at
/tmp/buildd/openldap-2.4.18/servers/slapd/connection.c:1261
#16 0x00007f05bb96500d in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:685
#17 0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#18 0x00007f05b99e55ad in clone () from /lib/libc.so.6
#19 0x0000000000000000 in ?? ()

Thread 2 (Thread 0x44f91950 (LWP 3251)):
#0  0x00007f05b9c73d29 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib/libpthread.so.0
#1  0x00007f05bb96650b in ldap_pvt_thread_cond_wait (cond=0x2210980,
mutex=0x2210958) at /tmp/buildd/openldap-2.4.18/libraries/libldap_r/thr_posix.c:277
#2  0x00007f05bb964f6f in ldap_int_thread_pool_wrapper (xpool=0x2210950) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/tpool.c:672
#3  0x00007f05b9c6ffc7 in start_thread () from /lib/libpthread.so.0
#4  0x00007f05b99e55ad in clone () from /lib/libc.so.6
#5  0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7f05bbdbc6e0 (LWP 3241)):
#0  0x00007f05b9c70715 in pthread_join () from /lib/libpthread.so.0
#1  0x00007f05bb966463 in ldap_pvt_thread_join (thread=1082726736,
thread_return=0x0) at
/tmp/buildd/openldap-2.4.18/libraries/libldap_r/thr_posix.c:197
#2  0x000000000043a035 in slapd_daemon () at
/tmp/buildd/openldap-2.4.18/servers/slapd/daemon.c:2835
#3  0x000000000041b0ed in main (argc=3, argv=0x7fffc3dcba38) at
/tmp/buildd/openldap-2.4.18/servers/slapd/main.c:950
#0  0x00007f05b9c70715 in pthread_join () from /lib/libpthread.so.0


Thread 3 appears to be the replication thread, which is hung.