[Date Prev][Date Next]
[Chronological]
[Thread]
[Top]
Re: (ITS#7974) LDBM's "laggard reader" flaw still present, in continue of ITS#7904
- To: openldap-its@OpenLDAP.org
- Subject: Re: (ITS#7974) LDBM's "laggard reader" flaw still present, in continue of ITS#7904
- From: leo@yuriev.ru
- Date: Thu, 23 Oct 2014 05:26:15 +0000
- Auto-submitted: auto-generated (OpenLDAP-ITS)
This is a multi-part message in MIME format.
--------------060806060106060808080102
Content-Type: text/plain; charset=windows-1251; format=flowed
Content-Transfer-Encoding: 7bit
The attached files is derived from OpenLDAP Software. All of the
modifications
to OpenLDAP Software represented in the following patch(es) were
developed by
Peter-Service LLC, Moscow, Russia. Peter-Service LLC has not assigned
rights
and/or interest in this work to any party. I, Leonid Yuriev am
authorized by
Peter-Service LLC, my employer, to release this work under the following
terms.
Peter-Service LLC hereby places the following modifications to OpenLDAP
Software
(and only these modifications) into the public domain. Hence, these
modifications may be freely used and/or redistributed for any purpose
with or
without attribution and/or other notice.
--------------060806060106060808080102
Content-Type: text/x-patch;
name="0001-lmdb-ITS-7974-a-reading-lag-for-dreamcatcher.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="0001-lmdb-ITS-7974-a-reading-lag-for-dreamcatcher.patch"
>From a30ece9b236b7217481c086fd27b133bd3404317 Mon Sep 17 00:00:00 2001
From: Leo Yuriev <leo@yuriev.ru>
Date: Tue, 21 Oct 2014 15:34:22 +0400
Subject: [PATCH 1/2] lmdb: ITS#7974 a reading lag for dreamcatcher.
---
libraries/liblmdb/lmdb.h | 11 +++++++++++
libraries/liblmdb/mdb.c | 20 ++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index a3ca62e..82eff14 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1571,6 +1571,17 @@ void mdb_env_set_oomkiller(MDB_env *env, MDB_oomkiller_func *oomkiller);
* @return A #MDB_oomkiller_func function or NULL if disabled.
*/
MDB_oomkiller_func* mdb_env_get_oomkiller(MDB_env *env);
+
+ /** @brief Returns a reading lag.
+ *
+ * Returns an information for estimate how much given read-only
+ * transaction is lagging relative the to actual head.
+ *
+ * @param[in] txn A transaction handle returned by #mdb_txn_begin()
+ * @param[out] percent Percentage of page allocation in the database.
+ * @return Number of transactions committed after the given was started for read, or -1 on failure.
+ */
+int mdb_txn_straggler(MDB_txn *txnm, int *percent);
/** @} */
#ifdef __cplusplus
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e60d83d..a417c9b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2823,6 +2823,26 @@ mdb_dbis_update(MDB_txn *txn, int keep)
env->me_numdbs = n;
}
+int
+mdb_txn_straggler(MDB_txn *txn, int *percent)
+{
+ MDB_env *env;
+ MDB_meta *meta;
+ txnid_t lag;
+
+ if (! txn || ! txn->mt_u.reader)
+ return -1;
+
+ env = txn->mt_env;
+ meta = env->me_metas[ mdb_env_pick_meta(env) ];
+ if (percent) {
+ long cent = env->me_maxpg / 100;
+ *percent = (meta->mm_last_pg + cent / 2 + 1) / (cent ? cent : 1);
+ }
+ lag = meta->mm_txnid - txn->mt_u.reader->mr_txnid;
+ return (0 > (int) lag) ? ~0u >> 1: lag;
+}
+
/** Common code for #mdb_txn_reset() and #mdb_txn_abort().
* May be called twice for readonly txns: First reset it, then abort.
* @param[in] txn the transaction handle to reset
--
2.1.0
--------------060806060106060808080102
Content-Type: text/x-patch;
name="0002-slapd-ITS-7974-dreamcatcher-feature.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="0002-slapd-ITS-7974-dreamcatcher-feature.patch"
>From 133cec8eadc93fe3083d110e58259ba2a067908c Mon Sep 17 00:00:00 2001
From: Leo Yuriev <leo@yuriev.ru>
Date: Tue, 21 Oct 2014 15:38:28 +0400
Subject: [PATCH 2/2] slapd: ITS#7974 dreamcatcher feature.
---
servers/slapd/back-mdb/config.c | 46 +++++++++++++++++++++++++++++-
servers/slapd/back-mdb/search.c | 62 ++++++++++++++++++++++-------------------
2 files changed, 79 insertions(+), 29 deletions(-)
diff --git a/servers/slapd/back-mdb/config.c b/servers/slapd/back-mdb/config.c
index b54da49..65034b1 100644
--- a/servers/slapd/back-mdb/config.c
+++ b/servers/slapd/back-mdb/config.c
@@ -39,7 +39,8 @@ enum {
MDB_MAXREADERS,
MDB_MAXSIZE,
MDB_MODE,
- MDB_SSTACK
+ MDB_SSTACK,
+ MDB_DREAMCATCHER
};
static ConfigTable mdbcfg[] = {
@@ -74,6 +75,10 @@ static ConfigTable mdbcfg[] = {
mdb_cf_gen, "( OLcfgDbAt:12.2 NAME 'olcDbMaxSize' "
"DESC 'Maximum size of DB in bytes' "
"SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL },
+ { "dreamcatcher", "lag> <percentage", 3, 3, 0, ARG_MAGIC|MDB_DREAMCATCHER,
+ mdb_cf_gen, "( OLcfgDbAt:12.4 NAME 'olcDbDreamcatcher' "
+ "DESC 'Dreamcatcher to avoids withhold of reclaiming' "
+ "SYNTAX OMsDirectoryString SINGLE-VALUE )",NULL, NULL },
{ "mode", "mode", 2, 2, 0, ARG_MAGIC|MDB_MODE,
mdb_cf_gen, "( OLcfgDbAt:0.3 NAME 'olcDbMode' "
"DESC 'Unix permissions of database files' "
@@ -319,6 +324,23 @@ mdb_cf_gen( ConfigArgs *c )
}
break;
+ case MDB_DREAMCATCHER:
+ if ( mdb->mi_renew_lag ) {
+ char buf[64];
+ struct berval bv;
+ bv.bv_len = snprintf( buf, sizeof(buf), "%ld %ld",
+ (long) mdb->mi_renew_lag, (long) mdb->mi_renew_percent );
+ if ( bv.bv_len > 0 && bv.bv_len < sizeof(buf) ) {
+ bv.bv_val = buf;
+ value_add_one( &c->rvalue_vals, &bv );
+ } else {
+ rc = 1;
+ }
+ } else {
+ rc = 1;
+ }
+ break;
+
case MDB_DIRECTORY:
if ( mdb->mi_dbenv_home ) {
c->value_string = ch_strdup( mdb->mi_dbenv_home );
@@ -389,6 +411,10 @@ mdb_cf_gen( ConfigArgs *c )
}
mdb->mi_txn_cp = 0;
break;
+ case MDB_DREAMCATCHER:
+ mdb->mi_renew_lag = 0;
+ mdb->mi_renew_percent = 0;
+ break;
case MDB_DIRECTORY:
mdb->mi_flags |= MDB_RE_OPEN;
ch_free( mdb->mi_dbenv_home );
@@ -574,6 +600,24 @@ mdb_cf_gen( ConfigArgs *c )
}
} break;
+ case MDB_DREAMCATCHER: {
+ long l;
+ if ( lutil_atolx( &l, c->argv[1], 0 ) != 0 || l < 1 ) {
+ fprintf( stderr, "%s: "
+ "invalid lag \"%s\" in \"dreamcatcher\".\n",
+ c->log, c->argv[1] );
+ return 1;
+ }
+ mdb->mi_renew_lag = l;
+ if ( lutil_atolx( &l, c->argv[2], 0 ) != 0 || l < 0 || l > 100 ) {
+ fprintf( stderr, "%s: "
+ "invalid percentage \"%s\" in \"dreamcatcher\".\n",
+ c->log, c->argv[2] );
+ return 1;
+ }
+ mdb->mi_renew_percent = l;
+ } break;
+
case MDB_DIRECTORY: {
FILE *f;
char *ptr, *testpath;
diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c
index 98b5f25..381cb80 100644
--- a/servers/slapd/back-mdb/search.c
+++ b/servers/slapd/back-mdb/search.c
@@ -340,22 +340,29 @@ typedef struct ww_ctx {
* case return an LDAP_BUSY error - let the client know this search
* couldn't succeed, but might succeed on a retry.
*/
+
+static void
+mdb_befree( Operation *op, ww_ctx *ww )
+{
+ assert(! ww->flag);
+ if ( ww->mcd ) {
+ MDB_val key, data;
+ mdb_cursor_get( ww->mcd, &key, &data, MDB_GET_CURRENT );
+ memcpy( &ww->key, key.mv_data, sizeof(ID) );
+ ww->data.mv_size = data.mv_size;
+ ww->data.mv_data = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx );
+ memcpy(ww->data.mv_data, data.mv_data, data.mv_size);
+ }
+ mdb_txn_reset( ww->txn );
+ ww->flag = 1;
+}
+
static void
mdb_writewait( Operation *op, slap_callback *sc )
{
ww_ctx *ww = sc->sc_private;
- if ( !ww->flag ) {
- if ( ww->mcd ) {
- MDB_val key, data;
- mdb_cursor_get( ww->mcd, &key, &data, MDB_GET_CURRENT );
- memcpy( &ww->key, key.mv_data, sizeof(ID) );
- ww->data.mv_size = data.mv_size;
- ww->data.mv_data = op->o_tmpalloc( data.mv_size, op->o_tmpmemctx );
- memcpy(ww->data.mv_data, data.mv_data, data.mv_size);
- }
- mdb_txn_reset( ww->txn );
- ww->flag = 1;
- }
+ if ( !ww->flag )
+ mdb_befree( op, ww );
}
static int
@@ -796,7 +803,6 @@ loop_begin:
goto done;
}
-
if ( nsubs < ncand ) {
unsigned i;
/* Is this entry in the candidate list? */
@@ -1034,14 +1040,6 @@ notfound:
ber_bvarray_free( erefs );
rs->sr_ref = NULL;
- if ( wwctx.flag ) {
- rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd );
- if ( rs->sr_err ) {
- send_ldap_result( op, rs );
- goto done;
- }
- }
-
goto loop_continue;
}
@@ -1096,13 +1094,6 @@ notfound:
}
goto done;
}
- if ( wwctx.flag ) {
- rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd );
- if ( rs->sr_err ) {
- send_ldap_result( op, rs );
- goto done;
- }
- }
}
} else {
@@ -1113,6 +1104,21 @@ notfound:
}
loop_continue:
+ if ( ! wwctx.flag && mdb->mi_renew_lag ) {
+ int percentage, lag = mdb_txn_straggler( ltid, &percentage );
+ if ( lag >= mdb->mi_renew_lag && percentage >= mdb->mi_renew_percent ) {
+ Debug( LDAP_DEBUG_TRACE, "dreamcather: lag %d, percentage %u%%\n", lag, percentage, 0 );
+ mdb_befree( op, &wwctx );
+ }
+ }
+ if ( wwctx.flag ) {
+ rs->sr_err = mdb_waitfixup( op, &wwctx, mci, mcd );
+ if ( rs->sr_err ) {
+ send_ldap_result( op, rs );
+ goto done;
+ }
+ }
+
if( e != NULL ) {
if ( e != base )
mdb_entry_return( op, e );
--
2.1.0
--------------060806060106060808080102--