(ITS#9060) lmdb corruption after many memory map resizes

[ruan.declercq@netronome.com]

Full_Name: Ruan de Clercq
Version: 
OS: Ubuntu 19.04
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (155.93.214.171)


Hi,

I am using 0.9.23-0ubuntu1 on ubuntu 19.04. 

I have an application where the exact db size isn't known beforehand. Therefore,
I set the memory map size beforehand, and then increase the size when I run out
of space. However, when I use lmdb in concurrent processes, and frequently
increase the size of the memory map, I eventually get a corrupt database
(MDB_CORRUPTED). Here's a minimal code example:

#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <time.h>
#include "lmdb.h"
#include <cstring>

int resize(MDB_env *env) {
  // read map size and increase
  int val;
  MDB_envinfo stat;
  MDB_txn *txn;
  if ((val = mdb_env_info(env, &stat) != MDB_SUCCESS) ||
      (val = mdb_env_set_mapsize(env, stat.me_mapsize + 4096) != MDB_SUCCESS)
||
      (val = mdb_env_info(env, &stat) != MDB_SUCCESS)) {
    printf("set new mapsize %d\n", val);
    return val;
  }
  printf("Resized to %lu\n", stat.me_mapsize);
  if (((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) ||
      ((val = mdb_txn_commit(txn)))) {
    printf("txn %d\n", val);
  }
  return val;
}

int main(int argc,char * argv[])
{
  timespec tp;
  clock_gettime(CLOCK_MONOTONIC, &tp);
  srand(tp.tv_sec + tp.tv_nsec);

  MDB_env *env = NULL;
  MDB_txn *txn = NULL;
  MDB_dbi dbi;
  int val;
  if ((val = mdb_env_create(&env) != MDB_SUCCESS) ||
      (val = mdb_env_set_maxreaders(env, 126) != MDB_SUCCESS)) {
    printf("Create error %d", val);
    goto out;
  }

  if ((val = mdb_env_set_mapsize(env, 4096 * 128) != MDB_SUCCESS) ||
      (val = mdb_env_set_maxdbs(env, 1) != MDB_SUCCESS) ||
      (val = mdb_env_open(env, "./data/", MDB_MAPASYNC | MDB_WRITEMAP, 0664) !=
MDB_SUCCESS) ||
      (val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS) ||
      (val = mdb_dbi_open(txn, "test", MDB_CREATE, &dbi) != MDB_SUCCESS) ||
      (val = mdb_txn_commit(txn) != MDB_SUCCESS)) {
    printf("create db %d\n", val);
    goto out;
  }

  for (int i = 0; i < 1000; i++) {
    char sval[32];
    sprintf(sval, "%03x %d foo bar", 32, rand());
    MDB_val mdb_key, mdb_val;
    mdb_key.mv_size = strlen(sval);;
    mdb_key.mv_data = sval;
    mdb_val.mv_size = strlen(sval);
    mdb_val.mv_data = sval;

    if ((val = mdb_txn_begin(env, NULL, 0, &txn) != MDB_SUCCESS)) {
      printf("mdb_txn_begin %d\n", val);
      goto out;
    }
    if ((val = mdb_put(txn, dbi, &mdb_key, &mdb_val, 0)) != MDB_SUCCESS) {
      mdb_txn_commit(txn);
      if (val == MDB_MAP_FULL) {
        val = resize(env);
        continue;
      }
      printf("mdb_put %d\n", val);
      goto out;
    }
    if ((val = mdb_txn_commit(txn)) != MDB_SUCCESS) {
      if (val == MDB_MAP_FULL) {
        val = resize(env);
        continue;
      }
      printf("mdb_txn_commit %d\n", val);
      goto out;
    }
  }
out:
  if (env) {
    mdb_dbi_close(env, dbi);
    mdb_env_close(env);
  }
  printf("OK\n");
}

After compiling I run the following command a couple of times:
for i in {1..20}; do echo $i; (lmdb_test &); done