[Date Prev][Date Next]
[Chronological]
[Thread]
[Top]
(ITS#7800) slapd with back-mdb segfaults periodically
Full_Name: Semyon Chaichenets
Version: 2.4.38
OS: Linux --- 3.2.0-58-generic #88-Ubuntu SMP Tue Dec 3 17:37:58 UTC 2013 x86_64
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (142.244.152.148)
We had slapd crash four times in the past 2 weeks, on two separate nodes.
Each time, the last instruction executed was back-mdb/dn2id.c:738, and each
time, it was called from back-mdb/search.c:747.
In every case, the cause appears to be an infinite loop: the stack is very deep
- in once instance, we had 2492641 frames in the back-trace; while the function
calls are oscillating between two addresses.
Both nodes in question had their database initialized from scratch by syncrepl.
I cannot provide core dumps for confidentiality reasons, but hopefully you will
find the following information helpful:
# ldd `which slapd`
linux-vdso.so.1 => (0x00007fff673ff000)
libldap_r-2.4.so.2 => /usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2
(0x00007f56feb1a000)
liblber-2.4.so.2 => /usr/lib/x86_64-linux-gnu/liblber-2.4.so.2
(0x00007f56fe90c000)
libuuid.so.1 => /lib/x86_64-linux-gnu/libuuid.so.1 (0x00007f56fe706000)
libslp.so.1 => /usr/lib/libslp.so.1 (0x00007f56fe4f5000)
libsasl2.so.2 => /usr/lib/x86_64-linux-gnu/libsasl2.so.2
(0x00007f56fe2da000)
libcrypt.so.1 => /lib/x86_64-linux-gnu/libcrypt.so.1
(0x00007f56fe0a0000)
libslapi-2.4.so.2 => /usr/lib/x86_64-linux-gnu/libslapi-2.4.so.2
(0x00007f56fde82000)
libltdl.so.7 => /usr/lib/x86_64-linux-gnu/libltdl.so.7
(0x00007f56fdc78000)
libwrap.so.0 => /lib/x86_64-linux-gnu/libwrap.so.0 (0x00007f56fda6e000)
libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0
(0x00007f56fd851000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f56fd491000)
libresolv.so.2 => /lib/x86_64-linux-gnu/libresolv.so.2
(0x00007f56fd274000)
libssl.so.1.0.0 => /lib/x86_64-linux-gnu/libssl.so.1.0.0
(0x00007f56fd016000)
libcrypto.so.1.0.0 => /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
(0x00007f56fcc3b000)
/lib64/ld-linux-x86-64.so.2 (0x00007f56ff0f6000)
libnsl.so.1 => /lib/x86_64-linux-gnu/libnsl.so.1 (0x00007f56fca20000)
libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f56fc81c000)
libz.so.1 => /lib/x86_64-linux-gnu/libz.so.1 (0x00007f56fc604000)
=== node 1, crash on 2014-02-07 ===
GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04
[..]
Reading symbols from /usr/sbin/slapd...done.
warning: exec file is newer than core file.
[New LWP 6117]
[New LWP 16603]
[New LWP 6106]
[New LWP 6116]
[New LWP 16602]
[New LWP 6115]
[New LWP 6114]
[New LWP 6113]
[New LWP 16604]
[New LWP 6112]
[..]
Core was generated by `/usr/sbin/slapd -u openldap -g openldap -h
ldap://127.0.0.1 ldap://-------'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007fe30c4818b1 in mdb_idscopes (op=0x7fe0d8000c30, isc=0x7fe0f94e9670)
at ../../../../../servers/slapd/back-mdb/dn2id.c:738
warning: Source file is more recent than executable.
738 isc->nrdns[isc->numrdns].bv_len = nrlen;
(gdb) info locals
mdb = <optimized out>
dbi = <optimized out>
key = {mv_size = 8, mv_data = 0x7fe0f9369588}
data = {mv_size = 38, mv_data = 0x7fe11e1e9df4}
id = 315367
id2 = {mid = 315054, mval = {mv_size = 34, mv_data = 0x7fe19e83f5e6}}
ptr = <optimized out>
rc = 1
x = <optimized out>
nrlen = 13
rlen = 13
d = 0x7fe11e1e9df4
(gdb) info registers
rax 0xd 13
rbx 0x7fe0f94e9670 140604232078960
rcx 0x130c990 19974544
rdx 0x7fe11e1e9df4 140604849692148
rsi 0x7fe0fa7fe000 140604252086272
rdi 0x7fe108147010 140604479926288
rbp 0x7fe0f9369588 0x7fe0f9369588
rsp 0x7fe0f9369540 0x7fe0f9369540
r8 0x26 38
r9 0x130c97 1248407
r10 0xd 13
r11 0x7fe11e1e9df6 140604849692150
r12 0x7fe0d8000c30 140603673283632
r13 0x1 1
r14 0x7fe0ec1029c0 140604009884096
r15 0x7fe0f94f98b0 140604232145072
rip 0x7fe30c4818b1 0x7fe30c4818b1 <mdb_idscopes+177>
eflags 0x10206 [ PF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
(gdb) info locals
scopeok = 0
edata = {mv_size = 13, mv_data = 0x7fe11e1e9e04}
mdb = <optimized out>
id = 1437644
cursor = 140604849692164
nsubs = 13
ncand = 1036510
cscope = 0
lastid = 140604849692164
candidates = {18446744073709551615, 1, 18446744073709551615, 208401, 208402,
270076, 292060, 528090, 528091, 528092, 528093, 528094, 528095, 528096, 528097,
528098, 528099, 617378, 687228, 1448040, 1448041,
524157, 524158, 524159, 574368, 656851, 810235, 864549, 84040, 84041, 94183,
94184, 118570, 132095, 132096, 132097, 132098, 132099, 132100, 132101, 132102,
132103, 132104, 132105, 132106, 132107, 142374,
142375, 149621, 150352, 175234, 175235, 175236, 175237, 175238, 175239,
175240, 175241, 175242, 175243, 175244, 175245, 175246, 185347, 185348, 205311,
236539, 242770, 242771, 242772, 242773, 242774, 242775,
242776, 242777, 242778, 242779, 242780, 242781, 242782, 253264, 253265,
282749, 308079, 308080, 308081, 308082, 308083, 308084, 308085, 308086, 308087,
308088, 308089, 308090, 308091, 318449, 318450, 404268,
404269, 620845, 665852, 665853, 665854, 665855, 665856, 665857, 665858,
665859, 665860, 665861, 665862, 665863, 665864, 677595, 677596, 836477, 1279141,
1287440, 1287441, 1287442, 1287443, 1287444, 1287445,
1287446, 1287447, 1287448, 1287449, 1287450, 1287451, 1294713, 1294714,
1303316, 1303317, 1303318, 1303319, 1303320, 1303321, 1303322, 1303323, 1303324,
1303325, 1303326, 1303327, 1310725, 1310726, 1319676,
1319677, 1319678, 1319679, 1319680, 1319681, 1319682, 1319683, 1319684,
1319685, 1319686, 1319687, 1327161, 1327162, 1336551, 1336552, 1336553, 1336554,
1336555, 1336556, 1336557, 1336558, 1336559, 1336560,
1336561, 1336562, 1343735, 1353674, 1353675, 1353676, 1353677, 1353678,
1353679, 1353680, 1353681, 1353682, 1353683, 1353684, 1353685, 1360913, 1371324,
1371325, 1371326, 1371327, 1371328, 1371329, 1371330,
1371331, 1371332, 1371333, 1371334, 1371335, 1378691, 1388937...}
iscopes = {0 <repeats 65502 times>, 140604009884096, 140604010617776, 635087,
140604479926288, 4294967298, 2, 12, 140606314528669, 9, 140606770728942, 12,
140604966947099, 9, 140606770728942, 14,
140606644291055, 24, 140606471143055, 14, 140606533666287, 14,
140606572504559, 0 <repeats 12 times>}
scopes = 0x7fe108147010
e = 0xd
base = 0x7fe11e1e9e04
matched = 0xd
attrs = <optimized out>
mask = <optimized out>
stoptime = 1391778836
manageDSAit = <optimized out>
isc = {mt = 0x7fe0ec1029c0, mc = 0x7fe0ec1db620, id = 1437644, scopes =
0x7fe108147010, numrdns = 1248406, nscope = 0, oscope = 2, rdns = {{bv_len =
14,
bv_val = 0x7fe1115e75a1 <Address 0x7fe1115e75a1 out of bounds>}, {bv_len =
11, bv_val = 0x7fe11e1e98d2 <Address 0x7fe11e1e98d2 out of bounds>}, {bv_len =
13,
bv_val = 0x7fe11e1e9e04 <Address 0x7fe11e1e9e04 out of bounds>} <repeats
2046 times>}, nrdns = {{bv_len = 13, bv_val = 0x7fe11e1e9e04 <Address
0x7fe11e1e9e04 out of bounds>} <repeats 2048 times>}}
mci = 0xd
mcd = 0x7fe11e1e9e04
opinfo = {moi_oe = {oe_next = {sle_next = 0xd}, oe_key = 0x7fe11e1e9e04},
moi_txn = 0xd, moi_ref = 505323012, moi_flag = -31 '\341'}
moi = 0xd
ltid = <optimized out>
(gdb) info registers
rax 0xd 13
rbx 0x7fe0d8000c30 140603673283632
rcx 0x130c990 19974544
rdx 0x7fe11e1e9df4 140604849692148
rsi 0x7fe0fa7fe000 140604252086272
rdi 0x7fe108147010 140604479926288
rbp 0x7fe0f94faa00 0x7fe0f94faa00
rsp 0x7fe0f93695c0 0x7fe0f93695c0
r8 0x26 38
r9 0x130c97 1248407
r10 0xd 13
r11 0x7fe11e1e9df6 140604849692150
r12 0x7fe0f94e96d0 140604232079056
r13 0x7fe0f94f96d0 140604232144592
r14 0x7fe0ec1029c0 140604009884096
r15 0x7fe0f94f98b0 140604232145072
rip 0x7fe30c47ab9b 0x7fe30c47ab9b <mdb_search+7451>
eflags 0x10206 [ PF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
(gdb) bt
#0 0x00007fe30c4818b1 in mdb_idscopes (op=0x7fe0d8000c30, isc=0x7fe0f94e9670)
at ../../../../../servers/slapd/back-mdb/dn2id.c:738
#1 0x00007fe30c47ab9b in mdb_search (op=0x7fe0d8000c30, rs=0x7fe0f94faa00) at
../../../../../servers/slapd/back-mdb/search.c:747
#2 0x00007fe11e1e9e04 in ?? ()
#3 0x000000000000000d in ?? ()
#4 0x00007fe11e1e9e04 in ?? ()
#5 0x000000000000000d in ?? ()
[..]
#2492634 0x00007fe11e1e9df6 in ?? ()
#2492635 0x000000000000000d in ?? ()
#2492636 0x00007fe11e1e9df6 in ?? ()
#2492637 0x000000000000000d in ?? ()
#2492638 0x00007fe11e1e9df6 in ?? ()
#2492639 0x000000000000000d in ?? ()
#2492640 0x00007fe11e1e9df6 in ?? ()
#2492641 0x0000000000000000 in ?? ()