(ITS#8530) LMDB: Error on committing large data

[newstzpz@gmail.com]

Full_Name: Peizhao Zhang
Version: LMDB (master)
OS: Windows 10
URL: ftp://ftp.openldap.org/incoming/
Submission from: (NULL) (47.218.255.229)


mdb_txn_commit will fail and returns error 87 when committing a huge data block
(512 * 412 * 4000 *  4 bytes). The following code demonstrates the problem:

#include <lmdb.h> 
#include <iostream>
#include <string>
using namespace std;

int main()
{
    string name = "./example.mdb";

    MDB_env* mdb_env;
    MDB_dbi mdb_dbi;
    MDB_val mdb_key, mdb_data;
    MDB_txn *mdb_txn;

    mdb_env_create(&mdb_env);
    int rc = mdb_env_open(mdb_env, name.c_str(), 0, 0664);
    size_t  map_size = 1ULL * 1024ULL * 1024ULL * 1024ULL * 30ULL;
    mdb_env_set_mapsize(mdb_env, map_size);

    mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
    mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi);

    int frame_size = 512 * 424 * sizeof(float);
    std::string frame;
    frame.resize(frame_size);

    for (int i = 0; i < 4000; i++) {
        std::string name = to_string(i);

        mdb_key.mv_size = name.size();
        mdb_key.mv_data = const_cast<char*>(name.data());
        mdb_data.mv_size = frame.size();
        mdb_data.mv_data = const_cast<char*>(frame.data());

        int put_rc = mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0);
        if (put_rc != MDB_SUCCESS) {
            cout << put_rc << endl;
            return -1;
        }
    }

    // FAIL IN THE FOLLOWING LINE
    rc = mdb_txn_commit(mdb_txn);
    if (rc != MDB_SUCCESS) {
        cout << "error on commit " << rc << endl;
        return -1;
    }

    mdb_dbi_close(mdb_env, mdb_dbi);A%A
    if (mdb_env != NULL) {
        mdb_dbi_close(mdb_env, mdb_dbi);
        mdb_env_close(mdb_env);
        mdb_env = NULL;
    }

    cout << "done " << endl;

    return 0;
}

After checking the debug information, it looks like the issue is caused by
overflow of the variable 'pos' defined in the function mdb_page_flush() (line
3521). 'pos' is defined as off_t, which actually is long in Windows. When
computing the value of 'pos' in line 3563 (pos = pgno * psize), it will overflow
when pgno =D3D 524434 and size == 4096, which is happened when large data blocks
are committed. This further causes WriteFile() in line 3581 to fail.

This issue looks fixed if the type of 'pos' is changed from 'off_t' to
'size_t'.