OpenLDAP
Up to top level
Build   Contrib   Development   Documentation   Historical   Incoming   Software Bugs   Software Enhancements   Web  

Logged in as guest

Viewing Software Bugs/7491
Full headers

From: clayton.stangeland@gmail.com
Subject: mdb write bus error before adding 5 million keys
Compose comment
Download message
State:
0 replies:
2 followups: 1 2

Major security issue: yes  no

Notes:

Notification:


Date: Fri, 11 Jan 2013 18:52:31 +0000
From: clayton.stangeland@gmail.com
To: openldap-its@OpenLDAP.org
Subject: mdb write bus error before adding 5 million keys
Full_Name: Clayton Stangeland
Version: git mdb.master last commit fed573cb86ed99f37bd062908ec814ee0ca47053
OS: Fedora 17 64 bit
URL: ftp://ftp.openldap.org/incoming/mtest-big.c
Submission from: (NULL) (192.94.73.31)


Building and running mdb from git branch mdb.master commit
fed573cb86ed99f37bd062908ec814ee0ca47053 on Fedora 17 64 bit I get a bus error.

When running the test below, which is a modification of mtest.c, I get a Bus
error. This happens while trying to write the file. gdb reports line 3936 of
mdb.c as the erroring line inside mdb_page_search_root: while (IS_BRANCH(mp)) {

If I set count to 4 million instead of 5 million it works, and adding back in
all the other mtest.c tests works as well then.

Here is the code:
/* mtest.c - memory-mapped database tester/toy */
/*
 * Copyright 2011 Howard Chu, Symas Corp.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */
#define _XOPEN_SOURCE 500               /* srandom(), random() */
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "lmdb.h"

int main(int argc,char * argv[])
{
        int i = 0, j = 0, rc;
        MDB_env *env;
        MDB_dbi dbi;
        MDB_val key, data;
        MDB_txn *txn;
        MDB_stat mst;
        MDB_cursor *cursor;
        int count;
        int *values;
        char sval[32];

        srandom(time(NULL));

            count = (5000000) + 64;
            values = (int *)malloc(count*sizeof(int));

            for(i = 0;i<count;i++) {
                        values[i] = random()%1000000000;
            }

                rc = mdb_env_create(&env);
                rc = mdb_env_set_mapsize(env, 10485760000);
                rc = mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/,
0664);
                rc = mdb_txn_begin(env, NULL, 0, &txn);
                rc = mdb_open(txn, NULL, 0, &dbi);

                key.mv_size = sizeof(sval);
                key.mv_data = sval;
                data.mv_size = sizeof(sval);
                data.mv_data = sval;

            for (i=0;i<count;i++) {
                        sprintf(sval, "%08x %d foo bar", values[i], values[i]);
                        rc = mdb_put(txn, dbi, &key, &data,
MDB_NOOVERWRITE);
                        if (rc) j++;
            }
                if (j) printf("%d duplicates skipped\n", j);
                rc = mdb_txn_commit(txn);
                rc = mdb_env_stat(env, &mst);

                free(values);
                mdb_close(env, dbi);
                mdb_env_close(env);

        return 0;

}

Followup 1

Download message
Date: Fri, 11 Jan 2013 11:49:05 -0800
From: Howard Chu <hyc@symas.com>
To: clayton.stangeland@gmail.com, openldap-its@openldap.org
Subject: Re: (ITS#7491) mdb write bus error before adding 5 million keys
clayton.stangeland@gmail.com wrote:
> Full_Name: Clayton Stangeland
> Version: git mdb.master last commit
fed573cb86ed99f37bd062908ec814ee0ca47053
> OS: Fedora 17 64 bit
> URL: ftp://ftp.openldap.org/incoming/mtest-big.c
> Submission from: (NULL) (192.94.73.31)
>
>
> Building and running mdb from git branch mdb.master commit
> fed573cb86ed99f37bd062908ec814ee0ca47053 on Fedora 17 64 bit I get a bus
error.

Thanks for the report. The bus error is now fixed in git mdb.master. We assume 
that most users will not batch so many operations into a single transaction.
>
> When running the test below, which is a modification of mtest.c, I get a
Bus
> error. This happens while trying to write the file. gdb reports line 3936
of
> mdb.c as the erroring line inside mdb_page_search_root: while
(IS_BRANCH(mp)) {
>
> If I set count to 4 million instead of 5 million it works, and adding back
in
> all the other mtest.c tests works as well then.
>
> Here is the code:
> /* mtest.c - memory-mapped database tester/toy */
> /*
>   * Copyright 2011 Howard Chu, Symas Corp.
>   * All rights reserved.
>   *
>   * Redistribution and use in source and binary forms, with or without
>   * modification, are permitted only as authorized by the OpenLDAP
>   * Public License.
>   *
>   * A copy of this license is available in the file LICENSE in the
>   * top-level directory of the distribution or, alternatively, at
>   * <http://www.OpenLDAP.org/license.html>.
>   */
> #define _XOPEN_SOURCE 500               /* srandom(), random() */
> #include <stdio.h>
> #include <stdlib.h>
> #include <time.h>
> #include "lmdb.h"
>
> int main(int argc,char * argv[])
> {
>          int i = 0, j = 0, rc;
>          MDB_env *env;
>          MDB_dbi dbi;
>          MDB_val key, data;
>          MDB_txn *txn;
>          MDB_stat mst;
>          MDB_cursor *cursor;
>          int count;
>          int *values;
>          char sval[32];
>
>          srandom(time(NULL));
>
>              count = (5000000) + 64;
>              values = (int *)malloc(count*sizeof(int));
>
>              for(i = 0;i<count;i++) {
>                          values[i] = random()%1000000000;
>              }
>
>                  rc = mdb_env_create(&env);
>                  rc = mdb_env_set_mapsize(env, 10485760000);
>                  rc = mdb_env_open(env, "./testdb", MDB_FIXEDMAP
/*|MDB_NOSYNC*/,
> 0664);
>                  rc = mdb_txn_begin(env, NULL, 0, &txn);
>                  rc = mdb_open(txn, NULL, 0, &dbi);
>
>                  key.mv_size = sizeof(sval);
>                  key.mv_data = sval;
>                  data.mv_size = sizeof(sval);
>                  data.mv_data = sval;
>
>              for (i=0;i<count;i++) {
>                          sprintf(sval, "%08x %d foo bar", values[i],
values[i]);
>                          rc = mdb_put(txn, dbi, &key, &data,
MDB_NOOVERWRITE);
>                          if (rc) j++;
>              }
>                  if (j) printf("%d duplicates skipped\n", j);
>                  rc = mdb_txn_commit(txn);
>                  rc = mdb_env_stat(env, &mst);
>
>                  free(values);
>                  mdb_close(env, dbi);
>                  mdb_env_close(env);
>
>          return 0;
>
> }
>
>


-- 
   -- Howard Chu
   CTO, Symas Corp.           http://www.symas.com
   Director, Highland Sun     http://highlandsun.com/hyc/
   Chief Architect, OpenLDAP  http://www.openldap.org/project/



Followup 2

Download message
Date: Fri, 11 Jan 2013 16:32:36 -0600
Subject: Re: (ITS#7491) mdb write bus error before adding 5 million keys
From: Clayton Stangeland <clayton.stangeland@gmail.com>
To: Howard Chu <hyc@symas.com>
Cc: openldap-its@openldap.org
Thank you. I did not consider the transaction size.

On 1/11/13, Howard Chu <hyc@symas.com> wrote:
> clayton.stangeland@gmail.com wrote:
>> Full_Name: Clayton Stangeland
>> Version: git mdb.master last commit
>> fed573cb86ed99f37bd062908ec814ee0ca47053
>> OS: Fedora 17 64 bit
>> URL: ftp://ftp.openldap.org/incoming/mtest-big.c
>> Submission from: (NULL) (192.94.73.31)
>>
>>
>> Building and running mdb from git branch mdb.master commit
>> fed573cb86ed99f37bd062908ec814ee0ca47053 on Fedora 17 64 bit I get a
bus
>> error.
>
> Thanks for the report. The bus error is now fixed in git mdb.master. We
> assume
> that most users will not batch so many operations into a single
> transaction.
>>
>> When running the test below, which is a modification of mtest.c, I get
a
>> Bus
>> error. This happens while trying to write the file. gdb reports line
3936
>> of
>> mdb.c as the erroring line inside mdb_page_search_root: while
>> (IS_BRANCH(mp)) {
>>
>> If I set count to 4 million instead of 5 million it works, and adding
back
>> in
>> all the other mtest.c tests works as well then.
>>
>> Here is the code:
>> /* mtest.c - memory-mapped database tester/toy */
>> /*
>>   * Copyright 2011 Howard Chu, Symas Corp.
>>   * All rights reserved.
>>   *
>>   * Redistribution and use in source and binary forms, with or without
>>   * modification, are permitted only as authorized by the OpenLDAP
>>   * Public License.
>>   *
>>   * A copy of this license is available in the file LICENSE in the
>>   * top-level directory of the distribution or, alternatively, at
>>   * <http://www.OpenLDAP.org/license.html>.
>>   */
>> #define _XOPEN_SOURCE 500               /* srandom(), random() */
>> #include <stdio.h>
>> #include <stdlib.h>
>> #include <time.h>
>> #include "lmdb.h"
>>
>> int main(int argc,char * argv[])
>> {
>>          int i = 0, j = 0, rc;
>>          MDB_env *env;
>>          MDB_dbi dbi;
>>          MDB_val key, data;
>>          MDB_txn *txn;
>>          MDB_stat mst;
>>          MDB_cursor *cursor;
>>          int count;
>>          int *values;
>>          char sval[32];
>>
>>          srandom(time(NULL));
>>
>>              count = (5000000) + 64;
>>              values = (int *)malloc(count*sizeof(int));
>>
>>              for(i = 0;i<count;i++) {
>>                          values[i] = random()%1000000000;
>>              }
>>
>>                  rc = mdb_env_create(&env);
>>                  rc = mdb_env_set_mapsize(env, 10485760000);
>>                  rc = mdb_env_open(env, "./testdb", MDB_FIXEDMAP
>> /*|MDB_NOSYNC*/,
>> 0664);
>>                  rc = mdb_txn_begin(env, NULL, 0, &txn);
>>                  rc = mdb_open(txn, NULL, 0, &dbi);
>>
>>                  key.mv_size = sizeof(sval);
>>                  key.mv_data = sval;
>>                  data.mv_size = sizeof(sval);
>>                  data.mv_data = sval;
>>
>>              for (i=0;i<count;i++) {
>>                          sprintf(sval, "%08x %d foo bar", values[i],
>> values[i]);
>>                          rc = mdb_put(txn, dbi, &key, &data,
>> MDB_NOOVERWRITE);
>>                          if (rc) j++;
>>              }
>>                  if (j) printf("%d duplicates skipped\n", j);
>>                  rc = mdb_txn_commit(txn);
>>                  rc = mdb_env_stat(env, &mst);
>>
>>                  free(values);
>>                  mdb_close(env, dbi);
>>                  mdb_env_close(env);
>>
>>          return 0;
>>
>> }
>>
>>
>
>
> --
>    -- Howard Chu
>    CTO, Symas Corp.           http://www.symas.com
>    Director, Highland Sun     http://highlandsun.com/hyc/
>    Chief Architect, OpenLDAP  http://www.openldap.org/project/
>


Up to top level
Build   Contrib   Development   Documentation   Historical   Incoming   Software Bugs   Software Enhancements   Web  

Logged in as guest


The OpenLDAP Issue Tracking System uses a hacked version of JitterBug

______________
© Copyright 2013, OpenLDAP Foundation, info@OpenLDAP.org