[Date Prev][Date Next]
[Chronological]
[Thread]
[Top]
(ITS#8429) deadlock on a modification operation with replication
Full_Name: Emmanuel Lecharny
Version: 2.4.44
OS: Linux CentOS 6
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (92.169.142.218)
I have a server blocked on a modification operation, which can neither be
shutdown nicely. Looking at the backtrace, I see that two threads are mutually
blocked (thread 2 and thread 3) :
Thread 4 (Thread 0x7f305fc97700 (LWP 28853))A%A
#0 0x0000003a9ba0b63c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f30a46395ab in ldap_pvt_thread_pool_destroy (tpool=0x727100,
run_pending=1) at /home/build/sold-2.4.44.2/openldap/libraries/libldap_r/tpool.c:817
#2 0x000000000042c619 in slapd_daemon_task (ptr=<value optimized out>) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/daemon.c:2851
#3 0x0000003a9ba07a51 in start_thread () from /lib64/libpthread.so.0
#4 0x0000003a9b6e896d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f305f496700 (LWP 28854)):
#0 0x0000003a9ba0b63c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f30a463839b in ldap_pvt_thread_rmutex_lock (rmutex=<value optimized
out>, owner=139845733803776) at
/home/build/sold-2.4.44.2/openldap/libraries/libldap_r/rmutex.c:129
#2 0x00007f30a09cf0cf in accesslog_op_mod (op=0x7f305f495450, rs=<value
optimized out>) at /home/build/sold-2.4.44.2/openldap/servers/slapd/overlays/accesslog.c:1984
#3 0x0000000000496dc2 in overlay_op_walk (op=0x7f305f495450, rs=0x7f305f494740,
which=op_modify, oi=0x886a30, on=0x88ba10) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/backover.c:661
#4 0x0000000000497873 in over_op_func (op=0x7f305f495450, rs=<value optimized
out>, which=<value optimized out>) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/backover.c:730
#5 0x000000000048bea6 in syncrepl_updateCookie (si=0x886650, op=0x7f305f495450,
syncCookie=<value optimized out>) at
/home/build/sold-2.4.44.2/enenldap/servers/slapd/syncrepl.c:3889
#6 0x000000000048cf1b in do_syncrep2 (op=0x7f305f495450, si=0x886650) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/syncrepl.c:1187
#7 0x00000000004932a2 in do_syncrepl (ctx=<value optimized out>, arg=0x885b50)
at /home/build/sold-2.4.44.2/openldap/servers/slapd/syncrepl.c:1561
#8 0x00007f30a463a058 in ldap_int_thread_pool_wrapper (xpool=0x817d80) at
/home/build/sold-2.4.44.2/openldap/libraries/libldap_r/tpool.c:956
#9 0x0000003a9ba07a51 in start_thread ()rfrom /lib62F2Flibpthread.so.0
#10 0x0000003a9b6e896d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f305ec95700 (LWP 28855)):
#0 0x0000003a9ba0b63c in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1 0x00007f30a463839b in ldap_pvt_thread_rmutex_lock (rmutex=<value optimized
out>, owner=139845725411072) at
/home/build/sold-2.4.44.2/openldap/libraries/libldap_r/rmutex.c:129
#2 0x00007f30a09cf0cf in accesslog_op_mod (op=0x7f3050114290, rs=<value
optimized out>) at /home/build/sold-2.4.44.2%2pepenldap/servers/slapd/overlays/accesslog.c:1984
#3 0x0000000000496dc2 in overlay_op_walk (op=0x7f3050114290, rs=0x7f305ec949f0,
which=op_modify, oi=0x886a30, on=0x88ba10) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/backover.c:661
#4 0x0000000000497873 in over_op_func (op=0x7f3050114290, rs=<value optimized
out>, which=<value optimized out>) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/backover.c:730
#5 0x00000000004464eb in fe_op_modify (op=0x7f3050114290, rs=0x7f305ec949f0) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/modify.c:303
#6 0x0000000000446e16 in do_modify (op=0x7f3050114290, rs=0x7f305ec949f0) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/modify.c:177
#7 0x000000000042ea79 in connection_operation (ctx=0x7f305ec94b70,
arg_v=0x7f3050114290) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/connection.c:1158
#8 0x000000000042f265 in connection_read_thread (ctx=0x7f305ec94b70,
argv=<value optimized out>) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/connection.c:1294
#9 0x00007f30a463a058 in ldap_int_thread_pool_wrapper (xpool=0x817d80) at
/home/build/sold-2.4.44.2/openldap/libraries/libldap_r/tpool.c:956
#10 0x0000003a9ba07a51 in start_thread () from /lib64/libpthread.so.0
#11 0x0000003a9b6e896d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f30a3565700 (LWP 28852)):
#0 0x0000003a9ba082ad in pthread_join () from /lib64/libpthread.so.0
#1 0x0000000000428fc9 in slapd_daemon () at
/home/build/sold-2.4.44.2/openldap/servers/apapd/daemon.c:2932
#2 0x0000000000415135 in main (argc=9, argv=<value optimized out>) at
/home/build/sold-2.4.44.2/openldap/servers/slapd/main.c:1016
This server is configured to use delta-syncrepl. Here is the slapd.conf file :
#-----------------------------------------------------------------------
# Main configuration
#
# MMR delta-syncrepl with TLS
#
# Server 1 will be replicated with Server 2 :
#
# +--------+ +--------+
# | Server |---[TLS]---->| Server |
# | 1 |<---[TLS]----| 2 |
# +--------+ +--------+
#
# See slapd.conf(5) for details on configuration options.
# This file should NOT be world readable.
#-----------------------------------------------------------------------
serverid 2
#-----------------------------------------------------------------------
# SCHEMA INCLUDES
# Use only those you need and leave the rest commented out.
include /opt/symas/etc/openldap/schema/core.schema
include /opt/symas/etc/openldap/schema/ppolicy.schema
include /opt/symas/etc/openldap/schema/cosine.schema
include /opt/symas/etc/openldap/schema/inetorgperson.schema
include /opt/symas/etc/openldap/schema/krb5-kdc.schema
include /opt/symas/etc/openldap/schema/rfc2307bis.schema
include /opt/symas/etc/openldap/schema/openssh.schema
#-----------------------------------------------------------------------
# PID / STARTUP ARGS
# Files in which to store the process id and startup arguments.
# These files are needed by the init scripts, so only change
# these if you are prepared to edit those scripts as well.
pidfile /var/symas/run/mmr-delta/slapd.pid
argsfile /var/symas/run/mmr-delta/slapd.args
#-----------------------------------------------------------------------
# TLS Setup Section
TLSCACertificateFile /opt/symas/ssl/mmr-delta/cacert.pem
TLSCertificateFile /opt/symas/ssl/mmr-delta/slapdcert.pem
TLSCertificateKeyFile /opt/symas/ssl/mmr-delta/slapdkey.pem
TLSCipherSuite HIGH:MEDIUM
TLSVerifyClient try
TLSProtocolMin 3.1
security ssf=128 tls=128
# This is the user that do the replication
authz-regexp "email=elecharny@symas.com,cn=([^,]*),ou=symas,o=symas corp,l=new
york,st=new jersey,c=us" "cn=replicator,dc=symas,dc=com"
#-----------------------------------------------------------------------
# Symas OpenLDAP supports threaded slapadd. This is only useful if running
# slapadd on a multi-cpu box. Generally, assign 1 thread per
# cpu, so if it is a 4 cpu box, use tool-threads 4. This
# specifically affects the creation of index databases, so if
# your database has fewer indices than CPUs, set it to the
# number of indices.
#tool-threads 2
#-----------------------------------------------------------------------
# MODULE PATH
# Choose the directory for loadable modules.
modulepath /opt/symas/lib64/openldap
# Uncomment the moduleloads as needed to enable additional
# functionalityi when configured. NOTE: We package many
# more modules options than those found below.
moduleload back_mdb.la
moduleload back_monitor.la
moduleload ppolicy.la
moduleload syncprov.la
moduleload accesslog.la
moduleload pw-sha2
D%D
#-----------------------------------------------------------------------
# Sample access control policy:
#-----------------------------------------------------------------------
access to dn="" by * read
# The replicator user has write access to the while DIT
# The users authenticated through a secured connection can read the DIT
access to dn.subtree="dc=symas,dc=com"
by dn.exact="cn=replicator,dc=symas,dc=com" write
# by * tls_ssf=128 read
# Allow self write access
# Allow auenenticated uss s read access
# Allow anonymous users to authenticate
access to *
by self write
by users read
by anonymous auth
# rootdn can always write!
#-----------------------------------------------------------------------
# LOGGING
loglevel stats sync
#######################################################################
# config database
#######################################################################
database config
rootdn "cn=admin,cn%coconfig"
rootpw {SSHA256}###################
#######################################################################
# Symas LMDB database definitions
#######################################################################%
dAdatabase mdb
suffix "dc=symas,dc=com"
rootdn "dc=symas,dc=com"
rootpw {SSHA256}#############
limits dn.exact="cn=replicator,dc=symas,dc=com" time.soft=unlimited
time.hard=unlimited size.soft=unlimited size.hard=unlimited
# Indices to maintain
index default eq
index objectClass
index cn eq,sub
index memberUID
index givenName eq,sub
index uniqueMember
index mail eq,sub
index entryUUID eq
index entryCSN eq
index uid eq,sub
directory /var/symas/openldap-data/mmr-delta/symas
maxsize 1073741824
#-----------------------------------------------------------------------
# SYNCREPL [LDAP1, server1]
syncrepl
rid=1
provider=ldap://<server1>:1389
bindmethod=sasl
saslmech=external
starttls=yes
tls_cacert=/opt/symas/ssl/mmr-delta/cacert.pem
tls_cert=/opt/symas/ssl/mmr-delta/slapdcert.pem
tls_key=/opt/symas/ssl/mmr-delta/slapdkey.pem
tls_reqcert=demand
type=refreshAndPersist
searchbase="dc=symas,dc=com"
filter="(objectclass=*)"
scope=sub
schemachecking=on
retry="5 10 60 +"
logbase="cn=accesslog"
loilteter="(&(objectClass=auditWriteObject)(reqResult=0))"
syncdata=accesslog
sizeLimit=unlimited
timelimit=unlimited
#
# SYNCREPL [LDAP2, server2]
syncrepl
rid=2
provider=ldap://<server2>:1389
bindmethod=sasl
saslmech=external
starttls=yes
tls_cacert=/opt/symas/ssl/mmr-delta/cacert.pem
tls_cert=/opt/symas/ssl/mmr-delta/slapdcert.pem
tls_key=/opt/symas/ssl/mmr-delta/slapdkey.pem
tls_reqcert=demand
type=refreshAndPersist
searchbase="dc=symas,dc=com"
filter="(objectclass=*)"
scope=sub
schemachecking=on
retry="5 10 60 +"
logbase="cn=accesslog"
logfilter="(&(objectClass=auditWriteObject9%9(reqResult=0))"
syncdata=accesslog
sizeLimit=unlimited
timelimit=unlimited
# ENABLE MIRROR MODE
mirrormode TRUE
#-----------------------------------------------------------------------
# Load an instance of the ppolicy overlay for the current database:
overlay ppolicy
# Specify the default password policy subentry to use when none is
# specified in an account's entry
ppolicy_default "cn=default,ou=Policies,dc=symas,dc=com"
#-----------------------------------------------------------------------
# Symas database overlays (syncprov and accesslog)
# OVERLAY [SYNCPROV]
overlay syncprov
syncprov-checkpoint 100 10
syncprov-sessionlog 10000
syncprov-reloadhint TRUE
#-----------------------------------------------------------------------
# OVERLAY [ACCESSLOG]
overlay accesslog
logdb cn=accesslog
logops writes
logsuccess TRUE
logpurge 24:00 01+00:00
########################################################"3%2#############
# AccessLog database
#######################################################################
database mdb
directory /var/symas/openldap-data/mmr-delta/accesslog
maxsize 5120000
suffix "cn=accesslog"
rootdn "cn=accesslog"
index default eq
index objectClass,entryCSN,entryUUID,reqEnd,reqResult,reqStart
access to *
by dn.base="cn=replicator,dc=symas,dc=com" read
by dn.base="dc=symas,dc=com" write
#-----------------------------------------------------------------------
# AccessLog overlay (syncprov)
overlay syncprov
syncprov-nopresent TRUE
syncprov-reloadhint TRUE
syncprov-checkpoint 100 10
syncprov-sessionlog 10000
limits dn.exact="cn=replicator,dc=symas,dc=com" time.soft=unlimited
time.hard=unlimited size.soft=unlimited size.hard=unlimited
#######################################################################
# Monitor database
#################################################################%#%3####
database monitor
When I try to shutdown the server nicely, the logs in syslog show :
May 19 04:18:15 cantal slapd[28852]: slapd shutdown: waiting for 2
operations/tasks to finish