Re: (ITS#8355) [lmdb] Issues with sorted duplicate DBs with cursor delete

[hhclaw.eb@gmail.com]

This is a multi-part message in MIME format.
--------------010901030100040802060200
Content-Type: text/plain; charset=windows-1252; format=flowed
Content-Transfer-Encoding: 7bit



On Sunday, January 24, 2016 07:44 PM, Howard Chu wrote:
> hhclaw.eb@gmail.com wrote:
>> Full_Name: H Law
>> Version: LMDB
>> OS: Linux
>> URL:
>> Submission from: (NULL) (42.2.241.129)
>>
>>
>> It seems that, after using a cursor delete during a cursor traversal 
>> on a dup
>> sort database, the cursor is in a strange state, when MDB_NEXT / 
>> MDB_NEXT_DUP
>> ceases to work properly while MDB_PREV /  MDB_PREV_DUP still functions.
>
> Thanks for the report. Fixed now in git.
>
Thank you. It now works except for the case when the deleted record is 
just before the end of the database.
In this case the next call to cursor get with MDB_NEXT / MDB_NEXT_DUP 
will still hang the program, instead of returning not found (or other 
errors).  (It seems to be stuck at some wait state without heavy cpu usage).

I attach a version of mtest3.c modified to show the issue.
>> In particular, when MDB_NEXT or MDB_NEXT_DUP is called after cursor 
>> deletion, if
>> next key/value pair exists, the cursor will not advance, and got 
>> stuck by
>> returning the same record when MDB_NEXT or MDB_NEXT_DUP is called 
>> repeatly.  In
>> case there is no next record, the program was hang.
>>
>> The following modified version of mtest3.c shows the issue.  I am 
>> testing this
>> on the latest commit 20dec1f69bf4860202c764ce92b1fbbe3d11a065 of lmdb 
>> on 20 Jan,
>> on x86-64 Linux.
>> I got a similar behaviour when a slightly earlier version of lmdb was
>> cross-compiled with a Java wrapper for use on Android, which is why I 
>> am testing
>> this.   The issue should therefore not be platform specific.
>
>
>


--------------010901030100040802060200
Content-Type: text/x-csrc;
 name="mtest3.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename="mtest3.c"

/* mtest3.c - memory-mapped database tester/toy */
/*
 * Copyright 2011-2015 Howard Chu, Symas Corp.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>.
 */

/* Tests for sorted duplicate DBs with cursor delete */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "lmdb.h"

#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))

int main(int argc,char * argv[])
{
	int i = 0, j = 0, rc;
	MDB_env *env;
	MDB_dbi dbi;
	MDB_val key, data;
	MDB_txn *txn;
	MDB_stat mst;
	MDB_cursor *cursor;
	int count;
	int *values;
	char sval[32];
	char kval[sizeof(int)];

	srand(time(NULL));

	memset(sval, 0, sizeof(sval));

        count = 10;
	values = (int *)malloc(count*sizeof(int));

	for(i = 0;i<count;i++) {
               values[i]= i * 10;
	}

	E(mdb_env_create(&env));
	E(mdb_env_set_mapsize(env, 10485760));
	E(mdb_env_set_maxdbs(env, 4));
	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));

	E(mdb_txn_begin(env, NULL, 0, &txn));
	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));

	key.mv_size = sizeof(int);
	key.mv_data = kval;
	data.mv_size = sizeof(sval);
	data.mv_data = sval;

	printf("Adding %d values\n", count);
	for (i=0;i<count;i++) {
		if (!(i & 0x07))
			sprintf(kval, "%03x", values[i]);
		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)))
			j++;
	}
	if (j) printf("%d duplicates skipped\n", j);
	E(mdb_txn_commit(txn));
	E(mdb_env_stat(env, &mst));


	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
		printf("key: %p %.*s, data: %p %.*s\n",
			key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
			data.mv_data, (int) data.mv_size, (char *) data.mv_data);
	}
	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
	mdb_cursor_close(cursor);
	mdb_txn_abort(txn);

	E(mdb_env_stat(env, &mst));
	E(mdb_txn_begin(env, NULL, 0, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
        /*
         * There are 2 items with key = values[8] at the end of the list
         */
	sprintf(kval, "%03x", values[8]);
	key.mv_size = sizeof(int);
	key.mv_data = kval;
	printf("\nCursor set / first dup\n");
	E(mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY));
	E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST_DUP));
	printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
	printf("Cursor next\n");
	j=0;
        /*
         * The program will hang at the following mdb_cursor_get 
         * after cursor delete
         */
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
                j++;
                if (j == 1) {
		    printf("delete the above key/data\n");
                    if (RES(MDB_NOTFOUND, mdb_cursor_del(cursor, 0))) {
	                mdb_cursor_close(cursor);
			mdb_txn_abort(txn);
                        break;
                    }
                }
                if (j > count) {
                   printf("Should not be there\n");
                   break;
                }
	}
        if (j > 1) {
	    mdb_cursor_close(cursor);
	    E(mdb_txn_commit(txn));
        } 


	E(mdb_txn_begin(env, NULL, 0, &txn));
	E(mdb_cursor_open(txn, dbi, &cursor));
	sprintf(kval, "%03x", values[0]);
	key.mv_size = sizeof(int);
	key.mv_data = kval;
	printf("\nCursor set / last dup\n");
	E(mdb_cursor_get(cursor, &key, &data, MDB_SET_KEY));
	E(mdb_cursor_get(cursor, &key, &data, MDB_LAST_DUP));
	printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);

	printf("Cursor prev\n");
        j=0;
	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV_DUP)) == 0) {
		printf("key: %.*s, data: %.*s\n",
			(int) key.mv_size,  (char *) key.mv_data,
			(int) data.mv_size, (char *) data.mv_data);
                j++;
                if (j == 1) {
		    printf("Delete the above key/data\n");
                    if (RES(MDB_NOTFOUND, mdb_cursor_del(cursor, 0))) {
	                mdb_cursor_close(cursor);
			mdb_txn_abort(txn);
                        break;
                    }
                }
                if (j > count) {
                   printf("Should not be there\n");
                   break;
                }
	}
        if (j > 1) {
	    mdb_cursor_close(cursor);
	    E(mdb_txn_commit(txn));
        } 

	mdb_dbi_close(env, dbi);
	mdb_env_close(env);
        free (values);
	return 0;
}

--------------010901030100040802060200--