[Date Prev][Date Next]
[Chronological]
[Thread]
[Top]
Re: (ITS#9060) lmdb corruption after many memory map resizes
- To: openldap-its@OpenLDAP.org
- Subject: Re: (ITS#9060) lmdb corruption after many memory map resizes
- From: hyc@symas.com
- Date: Tue, 06 Aug 2019 16:09:05 +0000
- Auto-submitted: auto-generated (OpenLDAP-ITS)
ruan.declercq@netronome.com wrote:
> Full_Name: Ruan de Clercq
> Version:
> OS: Ubuntu 19.04
> URL: ftp://ftp.openldap.org/incoming/
> Submission from: (NULL) (155.93.214.171)
>
>
> Hi,
>
> I am using 0.9.23-0ubuntu1 on ubuntu 19.04.
>
> I have an application where the exact db size isn't known beforehand. Therefore,
> I set the memory map size beforehand, and then increase the size when I run out
> of space.
This is incorrect usage of LMDB. The docs explicitly state to use a large size at
the beginning and leave it alone.
> However, when I use lmdb in concurrent processes, and frequently
> increase the size of the memory map, I eventually get a corrupt database
> (MDB_CORRUPTED).
Your code is incorrect. The docs explicitly state that there must not be any outstanding
activity when using the set_mapsize call.
Closing this ITS.
>
Here's a minimal code example:
>
> #include <stdio.h>
> #include <stdlib.h>
> #include <inttypes.h>
> #include <time.h>
> #include "lmdb.h"
> #include <cstring>
>
> int resize(MDB_env *env) {
> // read map size and increase
> int val;
> MDB_envinfo stat;
> MDB_txn *txn;
> if ((val = mdb_env_info(env, &stat) != MDB_SUCCESS) ||
> (val = mdb_env_set_mapsize(env, stat.me_mapsize + 4096) != MDB_SUCCESS)
> ||
> (val = mdb_env_info(env, &stat) != MDB_SUCCESS)) {
> printf("set new mapsize %d\n", val);
> return val;
> }
> printf("Resized to %lu\n", stat.me_mapsize);
> if (((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) ||
> ((val = mdb_txn_commit(txn)))) {
> printf("txn %d\n", val);
> }
> return val;
> }
>
> int main(int argc,char * argv[])
> {
> timespec tp;
> clock_gettime(CLOCK_MONOTONIC, &tp);
> srand(tp.tv_sec + tp.tv_nsec);
>
> MDB_env *env = NULL;
> MDB_txn *txn = NULL;
> MDB_dbi dbi;
> int val;
> if ((val = mdb_env_create(&env) != MDB_SUCCESS) ||
> (val = mdb_env_set_maxreaders(env, 126) != MDB_SUCCESS)) {
> printf("Create error %d", val);
> goto out;
> }
>
> if ((val = mdb_env_set_mapsize(env, 4096 * 128) != MDB_SUCCESS) ||
> (val = mdb_env_set_maxdbs(env, 1) != MDB_SUCCESS) ||
> (val = mdb_env_open(env, "./data/", MDB_MAPASYNC | MDB_WRITEMAP, 0664) !=
> MDB_SUCCESS) ||
> (val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS) ||
> (val = mdb_dbi_open(txn, "test", MDB_CREATE, &dbi) != MDB_SUCCESS) ||
> (val = mdb_txn_commit(txn) != MDB_SUCCESS)) {
> printf("create db %d\n", val);
> goto out;
> }
>
> for (int i = 0; i < 1000; i++) {
> char sval[32];
> sprintf(sval, "%03x %d foo bar", 32, rand());
> MDB_val mdb_key, mdb_val;
> mdb_key.mv_size = strlen(sval);;
> mdb_key.mv_data = sval;
> mdb_val.mv_size = strlen(sval);
> mdb_val.mv_data = sval;
>
> if ((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) {
> printf("mdb_txn_begin %d\n", val);
> goto out;
> }
> if ((val = mdb_put(txn, dbi, &mdb_key, &mdb_val, 0)) != MDB_SUCCESS) {
> mdb_txn_commit(txn);
> if (val == MDB_MAP_FULL) {
> val = resize(env);
> continue;
> }
> printf("mdb_put %d\n", val);
> goto out;
> }
> if ((val = mdb_txn_commit(txn)) != MDB_SUCCESS) {
> if (val == MDB_MAP_FULL) {
> val = resize(env);
> continue;
> }
> printf("mdb_txn_commit %d\n", val);
> goto out;
> }
> }
> out:
> if (env) {
> mdb_dbi_close(env, dbi);
> mdb_env_close(env);
> }
> printf("OK\n");
> }
>
> After compiling I run the following command a couple of times:
> for i in {1..20}; do echo $i; (lmdb_test &); done
>
>
--
-- Howard Chu
CTO, Symas Corp. http://www.symas.com
Director, Highland Sun http://highlandsun.com/hyc/
Chief Architect, OpenLDAP http://www.openldap.org/project/