[Date Prev][Date Next]
[Chronological]
[Thread]
[Top]
Re: Performance issue with In-Memory BDB
- To: "Howard Chu" <hyc@symas.com>
- Subject: Re: Performance issue with In-Memory BDB
- From: "Suhel Momin" <suhelmomin@gmail.com>
- Date: Tue, 4 Sep 2007 20:04:27 +0530
- Cc: openldap-software@openldap.org
- In-reply-to: <46D83160.8030108@symas.com>
- References: <7f1cc6540708310414u137066eco9c5ebc5682474b9f@mail.gmail.com> <46D83160.8030108@symas.com>
Sorry, but I could not understand the meaning of back-bdb residing in HEAD.
I have attached a diff over openldap-2.3.35 for changes regarding BDB in-memory.
This mainly has changes for keeping logs,environment and databases in memory.
The bdb in-memory code is under #ifdef BDB_INMEMORY
I am trying out different environment cache sizes to check any performance difference.
Thanks and Regards,
Suhel
On 8/31/07, Howard Chu <hyc@symas.com> wrote:
Suhel Momin wrote:
> Hi,
>
> I have made changes such that backend BDB ( ver 4.5 ) resides in
> memory instead of on disk . This is with default slapd configuration.
> I was hoping to have a better performance with this.
back-bdb as currently residing in HEAD already yields efficiencies of 95% of
available system bandwidth. It is extremely unlikely that you are going to be
able to improve the performance significantly, most changes you could make to
the code at this point will only yield performance losses.
> But what I now see
> is that do_add takes much more time then what it use to take when BDB
> was on-Disk. Indexing is done on objectclass in both the cases.
>
> Any pointers on why this could be an issue?
--
-- Howard Chu
Chief Architect, Symas Corp. http://www.symas.com
Director, Highland Sun
http://highlandsun.com/hyc/
Chief Architect, OpenLDAP http://www.openldap.org/project/
diff -Narup openldap-2.3.35/servers/slapd/back-bdb/dbcache.c inmemory_openldap-2.3.35/servers/slapd/back-bdb/dbcache.c
--- openldap-2.3.35/servers/slapd/back-bdb/dbcache.c 2007-01-03 03:14:00.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-bdb/dbcache.c 2007-09-04 19:05:57.000000000 +0530
@@ -116,6 +116,12 @@ bdb_db_cache(
file = ch_malloc( strlen( name ) + sizeof(BDB_SUFFIX) );
sprintf( file, "%s" BDB_SUFFIX, name );
+#ifdef BDB_INMEMORY
+ /* DB_MPOOL_NOFILE is set to specify database will remain in memory*/
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
+
+
#ifdef HAVE_EBCDIC
__atoe( file );
#endif
@@ -129,10 +135,16 @@ bdb_db_cache(
(SLAP_TOOL_QUICK|SLAP_TRUNCATE_MODE))
flags |= DB_TRUNCATE;
+#ifdef BDB_INMEMORY
+ rc = DB_OPEN( db->bdi_db,
+ NULL, /*No need to specify the name as the database will reside in-memory*/
+ NULL /* name */,
+ BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
+#else
rc = DB_OPEN( db->bdi_db,
file, NULL /* name */,
BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
-
+#endif
ch_free( file );
if( rc != 0 ) {
diff -Narup openldap-2.3.35/servers/slapd/back-bdb/init.c inmemory_openldap-2.3.35/servers/slapd/back-bdb/init.c
--- openldap-2.3.35/servers/slapd/back-bdb/init.c 2007-02-25 00:58:11.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-bdb/init.c 2007-09-04 19:01:25.000000000 +0530
@@ -58,10 +58,17 @@ bdb_db_init( BackendDB *be )
bdb = (struct bdb_info *) ch_calloc( 1, sizeof(struct bdb_info) );
/* DBEnv parameters */
+
+#ifdef BDB_INMEMORY
+ /* keep logs in memory*/
+ bdb->bi_dbenv_home = NULL;
+ bdb->bi_dbenv_xflags = DB_LOG_INMEMORY;
+ bdb->bi_dbenv_mode = 0;
+#else
bdb->bi_dbenv_home = ch_strdup( SLAPD_DEFAULT_DB_DIR );
bdb->bi_dbenv_xflags = 0;
bdb->bi_dbenv_mode = SLAPD_DEFAULT_DB_MODE;
-
+#endif
bdb->bi_cache.c_maxsize = DEFAULT_CACHE_SIZE;
bdb->bi_cache.c_minfree = 1;
@@ -278,8 +285,12 @@ shm_retry:
LDAP_XSTRING(bdb_db_open) ": dbenv_open(%s)\n",
bdb->bi_dbenv_home, 0, 0);
+#ifdef BDB_INMEMORY
+ /* DB_PRIVATE flag is added to keep environment in memory */
+ flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD | DB_PRIVATE;
+#else
flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD;
-
+#endif
if ( !quick )
flags |= BDB_TXN_FLAGS;
@@ -359,6 +370,9 @@ shm_retry:
goto fail;
}
+#ifdef BDB_INMEMORY
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
if( i == BDB_ID2ENTRY ) {
if ( slapMode & SLAP_TOOL_MODE )
db->bdi_db->mpf->set_priority( db->bdi_db->mpf,
@@ -391,6 +405,29 @@ shm_retry:
BDB_PAGESIZE );
}
+
+#ifdef BDB_INMEMORY
+
+#ifdef HAVE_EBCDIC
+ strcpy( path, bdbi_databases[i].file );
+ __atoe( path );
+ rc = DB_OPEN( db->bdi_db,
+ /* path*/NULL, /* don't need path for in-memory db*/
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#else
+ rc = DB_OPEN( db->bdi_db,
+ /* bdbi_databases[i].file*/NULL, /* don't need file for in-memory db */
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#endif
+
+#else
+
#ifdef HAVE_EBCDIC
strcpy( path, bdbi_databases[i].file );
__atoe( path );
@@ -409,6 +446,7 @@ shm_retry:
bdb->bi_dbenv_mode );
#endif
+#endif /*BDB_INMEMORY */
if ( rc != 0 ) {
char buf[SLAP_TEXT_BUFLEN];
diff -Narup openldap-2.3.35/servers/slapd/back-hdb/dbcache.c inmemory_openldap-2.3.35/servers/slapd/back-hdb/dbcache.c
--- openldap-2.3.35/servers/slapd/back-hdb/dbcache.c 2007-01-03 03:14:00.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-hdb/dbcache.c 2007-09-04 19:05:57.000000000 +0530
@@ -116,6 +116,12 @@ bdb_db_cache(
file = ch_malloc( strlen( name ) + sizeof(BDB_SUFFIX) );
sprintf( file, "%s" BDB_SUFFIX, name );
+#ifdef BDB_INMEMORY
+ /* DB_MPOOL_NOFILE is set to specify database will remain in memory*/
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
+
+
#ifdef HAVE_EBCDIC
__atoe( file );
#endif
@@ -129,10 +135,16 @@ bdb_db_cache(
(SLAP_TOOL_QUICK|SLAP_TRUNCATE_MODE))
flags |= DB_TRUNCATE;
+#ifdef BDB_INMEMORY
+ rc = DB_OPEN( db->bdi_db,
+ NULL, /*No need to specify the name as the database will reside in-memory*/
+ NULL /* name */,
+ BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
+#else
rc = DB_OPEN( db->bdi_db,
file, NULL /* name */,
BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode );
-
+#endif
ch_free( file );
if( rc != 0 ) {
diff -Narup openldap-2.3.35/servers/slapd/back-hdb/init.c inmemory_openldap-2.3.35/servers/slapd/back-hdb/init.c
--- openldap-2.3.35/servers/slapd/back-hdb/init.c 2007-02-25 00:58:11.000000000 +0530
+++ inmemory_openldap-2.3.35/servers/slapd/back-hdb/init.c 2007-09-04 19:01:25.000000000 +0530
@@ -58,10 +58,17 @@ bdb_db_init( BackendDB *be )
bdb = (struct bdb_info *) ch_calloc( 1, sizeof(struct bdb_info) );
/* DBEnv parameters */
+
+#ifdef BDB_INMEMORY
+ /* keep logs in memory*/
+ bdb->bi_dbenv_home = NULL;
+ bdb->bi_dbenv_xflags = DB_LOG_INMEMORY;
+ bdb->bi_dbenv_mode = 0;
+#else
bdb->bi_dbenv_home = ch_strdup( SLAPD_DEFAULT_DB_DIR );
bdb->bi_dbenv_xflags = 0;
bdb->bi_dbenv_mode = SLAPD_DEFAULT_DB_MODE;
-
+#endif
bdb->bi_cache.c_maxsize = DEFAULT_CACHE_SIZE;
bdb->bi_cache.c_minfree = 1;
@@ -278,8 +285,12 @@ shm_retry:
LDAP_XSTRING(bdb_db_open) ": dbenv_open(%s)\n",
bdb->bi_dbenv_home, 0, 0);
+#ifdef BDB_INMEMORY
+ /* DB_PRIVATE flag is added to keep environment in memory */
+ flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD | DB_PRIVATE;
+#else
flags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD;
-
+#endif
if ( !quick )
flags |= BDB_TXN_FLAGS;
@@ -359,6 +370,9 @@ shm_retry:
goto fail;
}
+#ifdef BDB_INMEMORY
+ db->bdi_db->mpf->set_flags(db->bdi_db->mpf,DB_MPOOL_NOFILE,1);
+#endif
if( i == BDB_ID2ENTRY ) {
if ( slapMode & SLAP_TOOL_MODE )
db->bdi_db->mpf->set_priority( db->bdi_db->mpf,
@@ -391,6 +405,29 @@ shm_retry:
BDB_PAGESIZE );
}
+
+#ifdef BDB_INMEMORY
+
+#ifdef HAVE_EBCDIC
+ strcpy( path, bdbi_databases[i].file );
+ __atoe( path );
+ rc = DB_OPEN( db->bdi_db,
+ /* path*/NULL, /* don't need path for in-memory db*/
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#else
+ rc = DB_OPEN( db->bdi_db,
+ /* bdbi_databases[i].file*/NULL, /* don't need file for in-memory db */
+ /* bdbi_databases[i].name, */ NULL,
+ bdbi_databases[i].type,
+ bdbi_databases[i].flags | flags,
+ bdb->bi_dbenv_mode );
+#endif
+
+#else
+
#ifdef HAVE_EBCDIC
strcpy( path, bdbi_databases[i].file );
__atoe( path );
@@ -409,6 +446,7 @@ shm_retry:
bdb->bi_dbenv_mode );
#endif
+#endif /*BDB_INMEMORY */
if ( rc != 0 ) {
char buf[SLAP_TEXT_BUFLEN];