Solaris dlapd core dumps - more clues

[Simon Spero <ses@tipper.oit.unc.edu>]

I've been poking around into some of the solaris  issues,I now have
a way to reliably make slapd dump core or lock up so I've hooked her up
to purify and started poking around. There seem to be some interesting
problems with freeing memory in the cache whilst it's still in use: a representative
chunk follows at the end of the message.

I can trigger the bug by deleting records; these records have already been pulled in as part of a previous search. The bug did not manifest itself when the field the search was performed on was not indexed; when the field is indexed the free frobbing begins in earnest.

There are a lot of FMR errors reported, but they all seem to be spin-offs of the same root cause, so I'm only including  one set of messages. I'm not real familar with the ldbm back end code, so I'm hoping someone who is will be able to look at the trace and go "duh".

Simon

      FMR: Free memory read (219702 times)
      This is occurring while in thread 17:
            cache_entrydn_cmp [cache.c:34]
               static int
               cache_entrydn_cmp( Entry *e1, Entry *e2 )
               {
            =>         return( strcasecmp( e1->e_dn, e2->e_dn ) );
               }

               static int
            ravl_delete    [avl.c:367]
                       if ( *root == NULLAVL )
                               return( 0 );

            =>         cmp = (*fcmp)( data, (*root)->avl_data );

                       /* found it! */
                       if ( cmp == 0 ) {
            avl_delete     [avl.c:453]
               {
                       int     shorter;

            =>         return( ravl_delete( root, data, fcmp, &shorter ) );
               }

               static
            cache_delete_entry_internal [cache.c:302]
               )
               {
                       /* dn tree */
            =>         if ( avl_delete( &cache->c_dntree, e, cache_entrydn_cmp ) == NULL ) {
                               return( -1 );
                       }

            cache_add_entry_lock [cache.c:172]
                                       e = cache->c_lrutail;

                                       /* delete from cache and lru q */
            =>                         rc = cache_delete_entry_internal( cache, e );

                                       entry_free( e );
                               }
            id2entry       [id2entry.c:141]

                       if ( (e = str2entry( data.dptr )) != NULL ) {
                               e->e_id = id;
            =>                 (void) cache_add_entry_lock( &li->li_cache, e, 0 );
                       }

                       ldbm_datum_free( db->dbc_db, data );
      Reading 4 bytes from 0x159b28 in the heap.
      Address 0x159b28 is at the beginning of a freed block of 28 bytes.
      This block was allocated from thread 14:
            malloc         [rtlib.o]
            calloc         [rtlib.o]
            ch_calloc      [ch_malloc.c:51]
               {
                       char    *new;

            =>         if ( (new = (char *) calloc( nelem, size )) == NULL ) {
                               Debug( LDAP_DEBUG_ANY, "calloc of %d elems of %d bytes failed\n",
                                 nelem, size, 0 );
                               exit( 1 );
            str2entry      [entry.c:48]

                       Debug( LDAP_DEBUG_TRACE, "=> str2entry\n", s, 0, 0 );

            =>         e = (Entry *) ch_calloc( 1, sizeof(Entry) );

                       /* check to see if there's an id included */
                       next = s;
            id2entry       [id2entry.c:139]
                               return( NULL );
                       }

            =>         if ( (e = str2entry( data.dptr )) != NULL ) {
                               e->e_id = id;
                               (void) cache_add_entry_lock( &li->li_cache, e, 0 );
                       }
            dn2entry       [dn2id.c:178]
                       char            *pdn;

                       if ( (id = dn2id( be, dn )) != NOID && (e = id2entry( be, id ))
            =>             != NULL ) {
                               return( e );
                       }
                       *matched = NULL;
      There have been 0 frees since this block was freed from thread 17:
            free           [rtlib.o]
            entry_free     [entry.c:197]
                               next = a->a_next;
                               attr_free( a );
                       }
            =>         free( e );
               }
            cache_add_entry_lock [cache.c:174]
                                       /* delete from cache and lru q */
                                       rc = cache_delete_entry_internal( cache, e );

            =>                         entry_free( e );
                               }
                       }

            id2entry       [id2entry.c:141]

                       if ( (e = str2entry( data.dptr )) != NULL ) {
                               e->e_id = id;
            =>                 (void) cache_add_entry_lock( &li->li_cache, e, 0 );
                       }

                       ldbm_datum_free( db->dbc_db, data );
            dn2entry       [dn2id.c:178]
                       char            *pdn;

                       if ( (id = dn2id( be, dn )) != NOID && (e = id2entry( be, id ))
            =>             != NULL ) {
                               return( e );
                       }
                       *matched = NULL;
            ldbm_back_delete [delete.c:25]
                       char            *matched = NULL;
                       Entry           *e;

            =>         if ( (e = dn2entry( be, dn, &matched )) == NULL ) {
                               send_ldap_result( conn, op, LDAP_NO_SUCH_OBJECT, matched, "" );
                               if ( matched != NULL ) {
                                       free( matched );