[Date Prev][Date Next] [Chronological] [Thread] [Top]

OL hangs in yield() (100% CPU) (ITS#2748)



Full_Name: Norbert Kiesel
Version: 2.1.22
OS: Solaris7
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (63.209.25.101)


I think this is a variant of the problem I reported on the mailing list on
2003-08-16. This time OL did not hang but spinned in yield(). OL did still
respond to login and serach requests with scope=0, but hang on all searches with
scope=1 or scope=2.  OL run for a 14 days with moderate usage (145492 searches
total) before it hang.

extract from syslog:
Oct  1 14:40:44 enterprise slapd[9668]: conn=1251 op=10 SRCH base="" scope=0
filter="(objectClass=*)"
Oct  1 14:40:44 enterprise slapd[9668]: conn=1251 op=10 SRCH
attr=namingcontexts
Oct  1 14:40:44 enterprise slapd[9668]: conn=1251 op=10 RESULT tag=101 err=0
text=
Oct  1 14:40:57 enterprise slapd[9668]: conn=1251 op=11 SRCH
base="o=tbdnetworks.com" scope=2
filter="(&(objectClass=tbddevice)(function=TT))"

enterprise# truss -p 9668 |& head -2
yield()                                         = 0
yield()                                         = 0

enterprise# pstack 9668
9668:   /opt/OpenLDAP/libexec/slapd -h ldap://0.0.0.0:389
-----------------  lwp# 1  --------------------------------
 ff199ddc lwp_cond_wait (fe7d7920, fe7d7908, 0)
 ff1917fc _lwp_cond_wait (fe7d7920, fe7d7908, 5651c08, bd, 6, fdb40dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d7908, 1, fdb40e48, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, d, 0, 1bd454c, 1, 1bacb9c) + e8
 00104780 __db_lget (1bd44e0, 0, 0, 1, 0, 1bacb9c) + 2d0
 00118148 __ham_lock_bucket (1bd44e0, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, fdb41078, 1, 0) + f8
 001155d4 __ham_item_next (1bd44e0, 1, fdb4124c, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (1bd44e0, fdb81388, ffffff, 1, fdb4124c, d)
 0010f6f0 ???????? (1, fdb81388, fdb81300, 1e, fdb4124c, 10f5f4)
 000fc500 __db_c_get (9daa90, fdb81388, fdb81300, 1e, fc110, 1bfcc00) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, fdb81388, 1d82e0, fdb8138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 54ceca0, 1d82e0, fdb816e4) + 6c
 0006e900 ???????? (0, fdb816e0, 2d82e0, 1d82e0, 14, 2582f0)
 0006db40 bdb_filter_candidates (0, fdb81700, 2d82e0, 1d82e0, 3582e0, 54cb689) +
2d4
 0006e07c ???????? (0, fdb81700, a1, 2582e0, 1d82e0, 2d82e0)
 0006da34 bdb_filter_candidates (0, fdb816f0, 2582e0, 1d82e0, 2d82e0, fdb81810)
+ 1c8
 0006e07c ???????? (0, fdb81710, a0, fdb81810, 1d82e0, 2582e0)
 0006da34 bdb_filter_candidates (0, fdb81720, fdb81810, 1d82e0, 2582e0,
fdb81700) + 1c8
 00065c38 ???????? (1d0848, 1d82e0, 9e8ef8, 54cedf0, 1, 0)
 00065740 bdb_search (0, 1d47d8, 2422820, fdc01ad8, fdc01ad0, 1) + fc0
 0002aed0 do_search (0, 2422820, fdc018c8, fdc01ac8, 165000, 0) + 740
 00029818 ???????? (241c100, 2422820, 29598, 0, 0, 0)
 0007da08 ???????? (175998, ff0f31a8, 0, 173ab0, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 2  --------------------------------
 ff1997f8 signotifywait ()
 ff0cf780 _dynamiclwps (ff0eca24, ff0ee38c, 2, 0, ff1b5e10, 0) + 1c
 ff0d26d8 thr_yield (0, 0, 0, 0, 0, 0) + 8c
-----------------  lwp# 3  --------------------------------
 000bcc64 __lock_get (1d21f0, 9e, 2, fc381558, 1, fc3817f8) + fc
 00068e88 bdb_cache_entry_db_lock (1d21f0, 9e, 1b74148, 0, 0, fc3817f8) + 34
 0006a2c4 bdb_cache_find_entry_id (1b74148, 1827a8, 16, 0, 9e, fc3817f8) + 8c
 00070040 bdb_id2entry_rw (1d1fc0, 0, 16, fc3817d8, 0, 9e) + 6c
 0006c424 bdb_dn2entry_rw (1d0848, 0, fc401ad0, fc3817d8, fc3817d4, 0) + 114
 000648d0 bdb_search (0, 1d5678, 1278cb0, fc401ad8, fc401ad0, 0) + 150
 0002aed0 do_search (0, 1278cb0, fc4018c8, fc401ac8, 165000, 0) + 740
 00029818 ???????? (1d18d8, 1278cb0, 29598, 0, 0, 0)
 0007da08 ???????? (175998, ff1d5d60, 0, fd401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 4  --------------------------------
 ff199ddc lwp_cond_wait (fe7d77b8, fe7d77a0, 0)
 ff1917fc _lwp_cond_wait (fe7d77b8, fe7d77a0, 5651b78, bd, 6, fd340dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d77a0, 1, fd340e48, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, c, 0, 1bd343c, 1, 9d955c) + e8
 00104780 __db_lget (1bd33d0, 0, 0, 1, 0, 9d955c) + 2d0
 00118148 __ham_lock_bucket (1bd33d0, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, fd341078, 1, 0) + f8
 001155d4 __ham_item_next (1bd33d0, 1, fd34124c, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (1bd33d0, fd381388, ffffff, 1, fd34124c, c)
 0010f6f0 ???????? (1, fd381388, fd381300, 1e, fd34124c, 10f5f4)
 000fc500 __db_c_get (9d9980, fd381388, fd381300, 1e, fc110, 1bfceb0) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, fd381388, a2c938, fd38138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 54cecb8, a2c938, fd3816e4) + 6c
 0006e900 ???????? (0, fd3816e0, b2c938, a2c938, 14, aac948)
 0006db40 bdb_filter_candidates (0, fd381700, b2c938, a2c938, bac938, 54bc261) +
2d4
 0006e07c ???????? (0, fd381700, a1, aac938, a2c938, b2c938)
 0006da34 bdb_filter_candidates (0, fd3816f0, aac938, a2c938, b2c938, fd381810)
+ 1c8
 0006e07c ???????? (0, fd381710, a0, fd381810, a2c938, aac938)
 0006da34 bdb_filter_candidates (0, fd381720, fd381810, a2c938, aac938,
fd381700) + 1c8
 00065c38 ???????? (1d0848, a2c938, 9e8ef8, 54ced18, 1, 0)
 00065740 bdb_search (0, 1d4a18, 2423e70, fd401ad8, fd401ad0, 1) + fc0
 0002aed0 do_search (0, 2423e70, fd4018c8, fd401ac8, ff0eca24, 0) + 740
 00029818 ???????? (241c400, 2423e70, 29598, 0, 0, 0)
 0007da08 ???????? (175998, ff1d5d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 5  --------------------------------
 ff199ddc lwp_cond_wait (fe7d6d68, fe7d6d50, 0)
 ff1917fc _lwp_cond_wait (fe7d6d68, fe7d6d50, 564f6e8, b6, 6, fcc00f44) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d6d50, 1, fcc00fd0, 11870c, 0) +
98
 000bd704 ???????? (fe7d59a0, 0, 0, 0, 2, 0)
 000bc32c __lock_vec (0, 800020ad, 0, fcc010f0, 2, fcc010ec) + 124
 001046ec __db_lget (1d2d68, 2, 0, 2, 0, fcc011b8) + 23c
 000e4c9c __bam_search (1, 0, 1d1fc0, 3202, 1, ff069556) + 840
 001260f8 ???????? (1d2d68, 1, fcc01558, 11, fcc012cc, 1)
 001249f0 ???????? (1d2d68, fcc01558, fcc01540, 11, fcc01364, 12478c)
 000fd3e0 __db_c_put (1d2d68, fcc01558, fcc01540, 11, fc940, 564e5b1) + aa0
 000f4ae4 __db_put (1d1fc0, 0, fcc01558, fcc01540, 0, f4774) + 370
 0006ff94 bdb_id2entry_put (0, 564d788, 1d1fc0, 0, 0, 1b74148) + 7c
 00061bc4 bdb_modify (0, 1d5e58, 1bfe088, fcc01ad8, fcc01ad0, 56349c8) + 408
 0003cd18 do_modify (0, 1bfe088, 173800, 132800, 165000, 0) + 7c0
 0002982c ???????? (1bf1298, 1bfe088, 29598, 0, 0, 0)
 0007da08 ???????? (175998, ff023d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 6  --------------------------------
 ff199ddc lwp_cond_wait (fe7d7650, fe7d7638, 0)
 ff1917fc _lwp_cond_wait (fe7d7650, fe7d7638, 5651b78, bd, 6, fb4011b4) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d7638, 1, fb401240, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d73e0, 0, 0, 0, 2, 0)
 000bcc50 __lock_get (1d21f0, 800020b8, 0, 1b59ebc, 2, 1b5a10c) + e8
 00104780 __db_lget (1b59e50, 0, 0, 2, 0, 1b5a10c) + 2d0
 00118148 __ham_lock_bucket (1b59e50, 2, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 2, ff06a45c, fb401470, ff0a12e8, ff09d080) + f8
 001155d4 __ham_item_next (1b59e50, 2, fb401644, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (1b59e50, fb401788, ffffff, 2, fb401644, 800020b8)
 0010f6f0 ???????? (1, fb401788, fb401700, 1e, fb401644, 10f5f4)
 000fc500 __db_c_get (1b59ff0, fb401788, fb401700, 1e, fc110, fb401714) + 3f0
 000717dc bdb_idl_delete_key (0, 1b59ad0, 564e720, fb401788, 12c4, fb401780) +
13c
 00072e9c bdb_key_change (1d0848, 1b59ad0, 564e720, 16681c, 12c4, 2) + e0
 00072840 ???????? (0, 564e720, 17f15c, 54d3700, 12c4, 2)
 00072954 ???????? (1d0848, 564e720, 17f120, 17d780, 54d3700, 12c4)
 000729cc bdb_index_values (1d0848, 564e720, 17d770, 54d3700, 12c4, 2) + 1c
 00072a9c bdb_index_entry (0, 564e720, 2, 54d3078, 54d46c8, 0) + c0
 0006bb98 bdb_delete (0, 1d5af8, 2424078, fb401ad8, fb401ad0, 0) + 800
 0003df44 do_delete (0, 2424078, 297ec, 4356, ff0eca24, 0) + 3d0
 000297f0 ???????? (241c1a0, 2424078, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe773d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 7  --------------------------------
 ff199ddc lwp_cond_wait (fe7d8028, fe7d8010, 0)
 ff1917fc _lwp_cond_wait (fe7d8028, fe7d8010, 56562c8, c8, 7, fbb40fc4) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d8010, 1, fbb41050, 11870c, 0) +
98
 000bd704 ???????? (fe7d6288, 0, 0, 0, 1, 0)
 000bc32c __lock_vec (0, cf, 0, fbb41170, 2, fbb4116c) + 124
 001046ec __db_lget (564f650, 2, 0, 1, 0, fbb41238) + 23c
 000e4c9c __bam_search (2c, 0, 1d2f48, 581, 1, ff076716) + 840
 001260f8 ???????? (564f650, 0, fbb814f8, 1e, fbb41344, 1d3210)
 00122f48 ???????? (564f650, fbb814f8, fbb81470, 1e, fbb413bc, 122a60)
 000fc500 __db_c_get (564f3f8, fbb814f8, fbb81470, 1e, fc110, 1d0488) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 1d2f48, 0, fbb814f8, 2ca5ac0, 564ffe1) +
16c
 0006d57c bdb_dn2idl (1d0848, 9d866c, 40, 2ca5ac0, 1, fbb817f8) + a4
 0006dd98 bdb_filter_candidates (0, fbb81710, 2ca5ac0, 2c25ac0, 2d25ac0,
fbb817f8) + 52c
 0006e07c ???????? (0, fbb81710, a0, fbb81810, 2c25ac0, 2ca5ac0)
 0006da34 bdb_filter_candidates (0, fbb81720, fbb81810, 2c25ac0, 2ca5ac0,
54d1c30) + 1c8
 00065c38 ???????? (1d0848, 2c25ac0, 9d8660, 54d1c30, 2, 3)
 00065740 bdb_search (0, 1d6098, 1d7f00, fbc01ad8, fbc01ad0, 2) + fc0
 0002aed0 do_search (0, 1d7f00, fbc018c8, fbc01ac8, ff0eca24, 0) + 740
 00029818 ???????? (241c340, 1d7f00, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe763d60, 1, ff0f978c, 0, fe763dc0)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 8  --------------------------------
 ff199ddc lwp_cond_wait (fe7d7a88, fe7d7a70, 0)
 ff1917fc _lwp_cond_wait (fe7d7a88, fe7d7a70, 5653ec0, bf, 6, f9340dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d7a70, 1, f9340e48, 11870c, 0) +
98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, c6, 0, 5651a2c, 1, 5651adc) + e8
 00104780 __db_lget (56519c0, 0, 0, 1, 0, 5651adc) + 2d0
 00118148 __ham_lock_bucket (56519c0, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, f9341078, 0, 0) + f8
 001155d4 __ham_item_next (56519c0, 1, f934124c, 0, 0, 0) + 10
 00111c24 ???????? (56519c0, f9381388, ffffff, 1, f934124c, c6)
 0010f6f0 ???????? (1, f9381388, f9381300, 1e, f934124c, 10f5f4)
 000fc500 __db_c_get (54d31c8, f9381388, f9381300, 1e, fc110, 54cdce8) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, f9381388, 4c25ae0, f938138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 54cee80, 4c25ae0, f93816e4) + 6c
 0006e900 ???????? (0, f93816e0, 4d25ae0, 4c25ae0, 0, 0)
 0006db40 bdb_filter_candidates (0, f9381700, 4d25ae0, 4c25ae0, 4da5ae0,
549e2e1) + 2d4
 0006e07c ???????? (0, f9381700, a1, 4ca5ae0, 4c25ae0, 4d25ae0)
 0006da34 bdb_filter_candidates (0, f93816f0, 4ca5ae0, 4c25ae0, 4d25ae0,
f9381810) + 1c8
 0006e07c ???????? (0, f9381710, a0, f9381810, 4c25ae0, 4ca5ae0)
 0006da34 bdb_filter_candidates (0, f9381720, f9381810, 4c25ae0, 4ca5ae0,
f9381700) + 1c8
 00065c38 ???????? (1d0848, 4c25ae0, 1b7a568, 54cedc0, 2, 0)
 00065740 bdb_search (0, 1d4fb8, 24233f0, f9401ad8, f9401ad0, 2) + fc0
 0002aed0 do_search (0, 24233f0, f94018c8, f9401ac8, 165000, 0) + 740
 00029818 ???????? (1d1678, 24233f0, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe753d60, 0, fac01e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 9  --------------------------------
 ff199ddc lwp_cond_wait (fe7d7ec0, fe7d7ea8, 0)
 ff1917fc _lwp_cond_wait (fe7d7ec0, fe7d7ea8, 5656290, c6, 7, f9b40fc4) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d7ea8, 1, f9b41050, 11870c, 0) +
98
 000bd704 ???????? (fe7d6288, 0, 0, 0, 1, 0)
 000bc32c __lock_vec (0, 4, 0, f9b41170, 2, f9b4116c) + 124
 001046ec __db_lget (9d82e8, 2, 0, 1, 0, f9b41238) + 23c
 000e4c9c __bam_search (2c, 0, 1d2f48, 581, 1, ff076716) + 840
 001260f8 ???????? (9d82e8, 0, f9b814f8, 1e, f9b41344, 1d3210)
 00122f48 ???????? (9d82e8, f9b814f8, f9b81470, 1e, f9b413bc, 122a60)
 000fc500 __db_c_get (1d3338, f9b814f8, f9b81470, 1e, fc110, 1d0488) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 1d2f48, 0, f9b814f8, 3ca5ad0, 54b6551) +
16c
 0006d57c bdb_dn2idl (1d0848, 9d866c, 40, 3ca5ad0, 0, 0) + a4
 0006dd98 bdb_filter_candidates (0, f9b81710, 3ca5ad0, 3c25ad0, 3d25ad0,
f9b817f8) + 52c
 0006e07c ???????? (0, f9b81710, a0, f9b81810, 3c25ad0, 3ca5ad0)
 0006da34 bdb_filter_candidates (0, f9b81720, f9b81810, 3c25ad0, 3ca5ad0,
54d26e0) + 1c8
 00065c38 ???????? (1d0848, 3c25ad0, 9d8660, 54d26e0, 2, 3)
 00065740 bdb_search (0, 1d5f78, 1269988, f9c01ad8, f9c01ad0, 2) + fc0
 0002aed0 do_search (0, 1269988, f9c018c8, f9c01ac8, ff0eca24, 0) + 740
 00029818 ???????? (1d1958, 1269988, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe743d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 10  --------------------------------
 ff199ddc lwp_cond_wait (fe7d7d58, fe7d7d40, 0)
 ff1917fc _lwp_cond_wait (fe7d7d58, fe7d7d40, 5656290, c6, 7, fa340fc4) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d7d40, 1, fa341050, 11870c, 0) +
98
 000bd704 ???????? (fe7d6288, 0, 0, 0, 1, 0)
 000bc32c __lock_vec (0, 8, 0, fa341170, 2, fa34116c) + 124
 001046ec __db_lget (1b598f0, 2, 0, 1, 0, fa341238) + 23c
 000e4c9c __bam_search (2c, 0, 1d2f48, 581, 1, ff076716) + 840
 001260f8 ???????? (1b598f0, 0, fa3814f8, 1e, fa341344, 1d3210)
 00122f48 ???????? (1b598f0, fa3814f8, fa381470, 1e, fa3413bc, 122a60)
 000fc500 __db_c_get (9d9110, fa3814f8, fa381470, 1e, fc110, 1d0488) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 1d2f48, 0, fa3814f8, 44a5ad8, 54cd649) +
16c
 0006d57c bdb_dn2idl (1d0848, 9d866c, 40, 44a5ad8, 0, 0) + a4
 0006dd98 bdb_filter_candidates (0, fa381710, 44a5ad8, 4425ad8, 4525ad8,
fa3817f8) + 52c
 0006e07c ???????? (0, fa381710, a0, fa381810, 4425ad8, 44a5ad8)
 0006da34 bdb_filter_candidates (0, fa381720, fa381810, 4425ad8, 44a5ad8,
54cf3a0) + 1c8
 00065c38 ???????? (1d0848, 4425ad8, 9d8660, 54cf3a0, 2, 3)
 00065740 bdb_search (0, 1d5d38, 2422be8, fa401ad8, fa401ad0, 2) + fc0
 0002aed0 do_search (0, 2422be8, fa4018c8, fa401ac8, ff0eca24, 0) + 740
 00029818 ???????? (241c1c0, 2422be8, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe743d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 11  --------------------------------
 ff199ddc lwp_cond_wait (fe7d8190, fe7d8178, 0)
 ff1917fc _lwp_cond_wait (fe7d8190, fe7d8178, 5656300, ca, 7, fab40fc4) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d8178, 1, fab41050, 11870c, 0) +
98
 000bd704 ???????? (fe7d6288, 0, 0, 0, 1, 0)
 000bc32c __lock_vec (0, d1, 0, fab41170, 2, fab4116c) + 124
 001046ec __db_lget (564fc58, 2, 0, 1, 0, fab41238) + 23c
 000e4c9c __bam_search (2c, 0, 1d2f48, 581, 1, ff076716) + 840
 001260f8 ???????? (564fc58, 0, fab814f8, 1e, fab41344, 1d3210)
 00122f48 ???????? (564fc58, fab814f8, fab81470, 1e, fab413bc, 122a60)
 000fc500 __db_c_get (564f958, fab814f8, fab81470, 1e, fc110, 1d0488) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 1d2f48, 0, fab814f8, 24a5ab8, 56500d1) +
16c
 0006d57c bdb_dn2idl (1d0848, 9d866c, 40, 24a5ab8, 1, fab817f8) + a4
 0006dd98 bdb_filter_candidates (0, fab81710, 24a5ab8, 2425ab8, 2525ab8,
fab817f8) + 52c
 0006e07c ???????? (0, fab81710, a0, fab81810, 2425ab8, 24a5ab8)
 0006da34 bdb_filter_candidates (0, fab81720, fab81810, 2425ab8, 24a5ab8,
fab81700) + 1c8
 00065c38 ???????? (1d0848, 2425ab8, 9d8660, 54d1990, 2, 0)
 00065740 bdb_search (0, 1d61b8, 2422e20, fac01ad8, fac01ad0, 2) + fc0
 0002aed0 do_search (0, 2422e20, fac018c8, fac01ac8, 165000, 0) + 740
 00029818 ???????? (241c280, 2422e20, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe753d60, 1, ff0f978c, 0, fe753dc0)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 12  --------------------------------
 ff199ddc lwp_cond_wait (fe7d86b8, fe7d86a0, 0)
 ff1917fc _lwp_cond_wait (fe7d86b8, fe7d86a0, 6659308, d6, 7, f8340dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d86a0, 1, f8340e48, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, de, 0, 6655744, 1, 665696c) + e8
 00104780 __db_lget (66556d8, 0, 0, 1, 0, 665696c) + 2d0
 00118148 __ham_lock_bucket (66556d8, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, f8341078, 1, 0) + f8
 001155d4 __ham_item_next (66556d8, 1, f834124c, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (66556d8, f8381388, ffffff, 1, f834124c, de)
 0010f6f0 ???????? (1, f8381388, f8381300, 1e, f834124c, 10f5f4)
 000fc500 __db_c_get (6656850, f8381388, f8381300, 1e, fc110, 564dca0) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, f8381388, 5e556d0, f838138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 54d2e80, 5e556d0, 54d2e6c) + 6c
 0006e900 ???????? (0, 54d2e68, 5f556d0, 5e556d0, 2c, 5ed56f8)
 0006db40 bdb_filter_candidates (0, 54d3060, 5f556d0, 5e556d0, 5fd56d0, 6657a19)
+ 2d4
 0006e07c ???????? (0, 54d3060, a0, 5ed56d0, 5e556d0, 5f556d0)
 0006da34 bdb_filter_candidates (0, 54d1b88, 5ed56d0, 5e556d0, 5f556d0,
f8381810) + 1c8
 0006e07c ???????? (0, f8381710, a0, f8381810, 5e556d0, 5ed56d0)
 0006da34 bdb_filter_candidates (0, f8381720, f8381810, 5e556d0, 5ed56d0,
54d1b88) + 1c8
 00065c38 ???????? (1d0848, 5e556d0, 1b884b8, 54d1b88, 1, 3)
 00065740 bdb_search (0, 1d63f8, 5654f30, f8401ad8, f8401ad0, 1) + fc0
 0002aed0 do_search (0, 5654f30, f84018c8, f8401ac8, ff0eca24, 0) + 740
 00029818 ???????? (5647bb8, 5654f30, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe713d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 13  --------------------------------
 ff199ddc lwp_cond_wait (fe7d8550, fe7d8538, 0)
 ff1917fc _lwp_cond_wait (fe7d8550, fe7d8538, 5e56c70, d3, 7, f8b40dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d8538, 1, f8b40e48, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, ce, 0, 54cd86c, 1, 564f174) + e8
 00104780 __db_lget (54cd800, 0, 0, 1, 0, 564f174) + 2d0
 00118148 __ham_lock_bucket (54cd800, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, f8b41078, 1, 0) + f8
 001155d4 __ham_item_next (54cd800, 1, f8b4124c, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (54cd800, f8b81388, ffffff, 1, f8b4124c, ce)
 0010f6f0 ???????? (1, f8b81388, f8b81300, 1e, f8b4124c, 10f5f4)
 000fc500 __db_c_get (54cd970, f8b81388, f8b81300, 1e, fc110, 564dbd0) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, f8b81388, 5654fd0, f8b8138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 54d2518, 5654fd0, 54cffd4) + 6c
 0006e900 ???????? (0, 54cffd0, 5754fd0, 5654fd0, 2c, 56d4ff8)
 0006db40 bdb_filter_candidates (0, 54d1a08, 5754fd0, 5654fd0, 57d4fd0, 5654ec1)
+ 2d4
 0006e07c ???????? (0, 54d1a08, a0, 56d4fd0, 5654fd0, 5754fd0)
 0006da34 bdb_filter_candidates (0, 54d1948, 56d4fd0, 5654fd0, 5754fd0,
f8b81810) + 1c8
 0006e07c ???????? (0, f8b81710, a0, f8b81810, 5654fd0, 56d4fd0)
 0006da34 bdb_filter_candidates (0, f8b81720, f8b81810, 5654fd0, 56d4fd0,
54d1948) + 1c8
 00065c38 ???????? (1d0848, 5654fd0, 1b884b8, 54d1948, 1, 3)
 00065740 bdb_search (0, 1d62d8, 564fe70, f8c01ad8, f8c01ad0, 1) + fc0
 0002aed0 do_search (0, 564fe70, f8c018c8, f8c01ac8, ff0eca24, 0) + 740
 00029818 ???????? (5647b78, 564fe70, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe703d60, 1, ff0f978c, 0, fe703dc0)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 14  --------------------------------
 ff199ddc lwp_cond_wait (fe7d8820, fe7d8808, 0)
 ff1917fc _lwp_cond_wait (fe7d8820, fe7d8808, 6e5bdb0, d9, 7, f7b40dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d8808, 1, f7b40e48, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, e1, 0, 6e57da4, 1, 6e58fcc) + e8
 00104780 __db_lget (6e57d38, 0, 0, 1, 0, 6e58fcc) + 2d0
 00118148 __ham_lock_bucket (6e57d38, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, f7b41078, 1, 0) + f8
 001155d4 __ham_item_next (6e57d38, 1, f7b4124c, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (6e57d38, f7b81388, ffffff, 1, f7b4124c, e1)
 0010f6f0 ???????? (1, f7b81388, f7b81300, 1e, f7b4124c, 10f5f4)
 000fc500 __db_c_get (6e58eb0, f7b81388, f7b81300, 1e, fc110, 6e5a170) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, f7b81388, 6657d30, f7b8138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 54d2c88, 6657d30, 54d3374) + 6c
 0006e900 ???????? (0, 54d3370, 6757d30, 6657d30, 2c, 66d7d58)
 0006db40 bdb_filter_candidates (0, 54d2d18, 6757d30, 6657d30, 67d7d30, 6657a19)
+ 2d4
 0006e07c ???????? (0, 54d2d18, a0, 66d7d30, 6657d30, 6757d30)
 0006da34 bdb_filter_candidates (0, 54d2d60, 66d7d30, 6657d30, 6757d30,
f7b81810) + 1c8
 0006e07c ???????? (0, f7b81710, a0, f7b81810, 6657d30, 66d7d30)
 0006da34 bdb_filter_candidates (0, f7b81720, f7b81810, 6657d30, 66d7d30,
54d2d60) + 1c8
 00065c38 ???????? (1d0848, 6657d30, 1b884b8, 54d2d60, 1, 3)
 00065740 bdb_search (0, 1d6518, 6657b98, f7c01ad8, f7c01ad0, 1) + fc0
 0002aed0 do_search (0, 6657b98, f7c018c8, f7c01ac8, ff0eca24, 0) + 740
 00029818 ???????? (5647d78, 6657b98, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe6f3d60, 0, fe401e50, 1, fe401000)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 15  --------------------------------
 ff198044 poll     (fe4011b8, 13, ffffffff)
 ff14f8c8 select   (fe401250, fe401c50, fe401cd0, 0, 13, ffffffff) + 334
 ff0db690 select   (1, 2b, 165000, 0, 166c00, ff1d5dc0) + 34
 ff0dbcf8 _thread_start (0, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 16  --------------------------------
 ff199ddc lwp_cond_wait (fe7d8988, fe7d8970, 0)
 ff1917fc _lwp_cond_wait (fe7d8988, fe7d8970, 765e288, dc, 7, f7340dbc) + c
 000df9a0 __db_pthread_mutex_lock (1d21f0, fe7d8970, 1, f7340e48, 11870c,
10f5f4) + 98
 000bd704 ???????? (fe7d7548, 0, 0, 0, 1, 0)
 000bcc50 __lock_get (1d21f0, e3, 0, 765b964, 1, 765ba14) + e8
 00104780 __db_lget (765b8f8, 0, 0, 1, 0, 765ba14) + 2d0
 00118148 __ham_lock_bucket (765b8f8, 1, 0, 0, 0, 0) + 88
 00117f24 __ham_get_cpage (0, 1, ff061224, f7341078, 1, 0) + f8
 001155d4 __ham_item_next (765b8f8, 1, f734124c, ff0a1f60, 1, 0) + 10
 00111c24 ???????? (765b8f8, f7381388, ffffff, 1, f734124c, e3)
 0010f6f0 ???????? (1, f7381388, f7381300, 1e, f734124c, 10f5f4)
 000fc500 __db_c_get (765a7e8, f7381388, f7381300, 1e, fc110, 6e5a230) + 3f0
 000706ec bdb_idl_fetch_key (fffbffa4, 9d92f0, 0, f7381388, 6e5a670, f738138c) +
16c
 00072c5c bdb_key_read (1d0848, 9d92f0, 0, 24019f8, 6e5a670, f73816e4) + 6c
 0006e900 ???????? (0, f73816e0, 6f5a670, 6e5a670, 0, 0)
 0006db40 bdb_filter_candidates (0, f7381700, 6f5a670, 6e5a670, 6fda670, 0) +
2d4
 0006e07c ???????? (0, f7381700, a1, 6eda670, 6e5a670, 6f5a670)
 0006da34 bdb_filter_candidates (0, f73816f0, 6eda670, 6e5a670, 6f5a670,
f7381810) + 1c8
 0006e07c ???????? (0, f7381710, a0, f7381810, 6e5a670, 6eda670)
 0006da34 bdb_filter_candidates (0, f7381720, f7381810, 6e5a670, 6eda670,
f7381700) + 1c8
 00065c38 ???????? (1d0848, 6e5a670, 1d8150, 54d2cd0, 2, 0)
 00065740 bdb_search (0, 1d6638, 6e5a5e8, f7401ad8, f7401ad0, 2) + fc0
 0002aed0 do_search (0, 6e5a5e8, f74018c8, f7401ac8, ff0eca24, 0) + 740
 00029818 ???????? (5647e38, 6e5a5e8, 29598, 0, 0, 0)
 0007da08 ???????? (175998, fe6d3d60, 1, ff0f978c, 0, fe6d3dc0)
 ff0dbcf8 _thread_start (175998, 0, 0, 0, 0, 0) + 40
-----------------  lwp# 17  --------------------------------
 ff199ddc lwp_cond_wait (ff0f3f98, ff0f3fa8, fe6c3c90)
 ff1918a8 _lwp_cond_timedwait (ff0f3f98, ff0f3fa8, 0, 3f7b4d3b, 0, 0) + 90
 ff0ca94c _age     (ff0eca24, ff0ed7a4, ff0ee078, ff0ee090, 3, ff0eca24) + 90
 ff0cbecc _lwp_start (4000, ff00fc7c, ff0ed7f0, ff0ed7d8, ff0eca24, ff00fb70) +
14
 ff0d26d8 thr_yield (0, 0, 0, 0, 0, 0) + 8c