--- servers/slapd/overlays/syncprov.c 2004/11/24 16:07:14 1.9 +++ servers/slapd/overlays/syncprov.c 2004/11/26 08:40:22 1.14 @@ -19,8 +19,6 @@ #include "portable.h" -#define SLAPD_OVER_SYNCPROV SLAPD_MOD_DYNAMIC - #ifdef SLAPD_OVER_SYNCPROV #include @@ -37,6 +35,8 @@ typedef struct syncops { int s_flags; /* search status */ } syncops; +static int sync_cid; + #define PS_IS_REFRESHING 0x01 /* Record of which searches matched at premodify step */ @@ -51,6 +51,7 @@ typedef struct syncprov_info_t { int si_gotcsn; /* is our ctxcsn up to date? */ ldap_pvt_thread_mutex_t si_csn_mutex; ldap_pvt_thread_mutex_t si_ops_mutex; + char si_ctxcsnbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; } syncprov_info_t; typedef struct opcookie { @@ -64,26 +65,24 @@ typedef struct opcookie { } opcookie; typedef struct fbase_cookie { - struct berval *fdn; - syncops *fss; - int fbase; - int fsuffix; + struct berval *fdn; /* DN of a modified entry, for scope testing */ + syncops *fss; /* persistent search we're testing against */ + int fbase; /* if TRUE we found the search base and it's still valid */ + int fscope; /* if TRUE then fdn is within the psearch scope */ } fbase_cookie; static AttributeName csn_anlist[2]; static AttributeName uuid_anlist[2]; -static int -dn_avl_cmp( const void *c1, const void *c2 ) -{ - struct berval *bv1 = (struct berval *)c1; - struct berval *bv2 = (struct berval *)c2; - int rc = bv1->bv_len - bv2->bv_len; - - if ( rc ) return rc; - return ber_bvcmp( bv1, bv2 ); -} - +/* syncprov_findbase: + * finds the true DN of the base of a search (with alias dereferencing) and + * checks to make sure the base entry doesn't get replaced with a different + * entry (e.g., swapping trees via ModDN, or retargeting an alias). If a + * change is detected, any persistent search on this base must be terminated / + * reloaded. + * On the first call, we just save the DN and entryID. On subsequent calls + * we compare the DN and entryID with the saved values. + */ static int findbase_cb( Operation *op, SlapReply *rs ) { @@ -99,17 +98,40 @@ findbase_cb( Operation *op, SlapReply *r fc->fbase = 1; fc->fss->s_eid = rs->sr_entry->e_id; ber_dupbv( &fc->fss->s_base, &rs->sr_entry->e_nname ); + } else if ( rs->sr_entry->e_id == fc->fss->s_eid && dn_match( &rs->sr_entry->e_nname, &fc->fss->s_base )) { + + /* OK, the DN is the same and the entryID is the same. Now + * see if the fdn resides in the scope. + */ fc->fbase = 1; - fc->fsuffix = dnIsSuffix( fc->fdn, &rs->sr_entry->e_nname ); + switch ( fc->fss->s_op->ors_scope ) { + case LDAP_SCOPE_BASE: + fc->fscope = dn_match( fc->fdn, &rs->sr_entry->e_nname ); + break; + case LDAP_SCOPE_ONELEVEL: { + struct berval pdn; + dnParent( fc->fdn, &pdn ); + fc->fscope = dn_match( &pdn, &rs->sr_entry->e_nname ); + break; } + case LDAP_SCOPE_SUBTREE: + fc->fscope = dnIsSuffix( fc->fdn, &rs->sr_entry->e_nname ); + break; +#ifdef LDAP_SCOPE_SUBORDINATE + case LDAP_SCOPE_SUBORDINATE: + fc->fscope = dnIsSuffix( fc->fdn, &rs->sr_entry->e_nname ) && + !dn_match( fc->fdn, &rs->sr_entry->e_nname ); + break; +#endif + } } } return LDAP_SUCCESS; } static int -syncprov_findbase( Operation *op, syncops *ss, fbase_cookie *fc ) +syncprov_findbase( Operation *op, fbase_cookie *fc ) { opcookie *opc = op->o_callback->sc_private; slap_overinst *on = opc->son; @@ -125,19 +147,19 @@ syncprov_findbase( Operation *op, syncop cb.sc_response = findbase_cb; cb.sc_private = fc; - fop.o_sync_mode = 0; + fop.o_sync_mode &= SLAP_CONTROL_MASK; /* turn off sync mode */ fop.o_callback = &cb; fop.o_tag = LDAP_REQ_SEARCH; fop.ors_scope = LDAP_SCOPE_BASE; - fop.ors_deref = ss->s_op->ors_deref; + fop.ors_deref = fc->fss->s_op->ors_deref; fop.ors_slimit = 1; fop.ors_tlimit = SLAP_NO_LIMIT; fop.ors_attrs = slap_anlist_no_attrs; fop.ors_attrsonly = 1; - fop.ors_filter = ss->s_op->ors_filter; - fop.ors_filterstr = ss->s_op->ors_filterstr; + fop.ors_filter = fc->fss->s_op->ors_filter; + fop.ors_filterstr = fc->fss->s_op->ors_filterstr; - fop.o_req_ndn = ss->s_op->o_req_ndn; + fop.o_req_ndn = fc->fss->s_op->o_req_ndn; fop.o_bd->bd_info = on->on_info->oi_orig; rc = fop.o_bd->be_search( &fop, &frs ); @@ -151,6 +173,26 @@ syncprov_findbase( Operation *op, syncop return LDAP_NO_SUCH_OBJECT; } +/* syncprov_findcsn: + * This function has three different purposes, but they all use a search + * that filters on entryCSN so they're combined here. + * 1: when the current contextCSN is unknown (i.e., at server start time) + * and a syncrepl search has arrived with a cookie, we search for all entries + * with CSN >= the cookie CSN, and store the maximum as our contextCSN. Also, + * we expect to find the cookie CSN in the search results, and note if we did + * or not. If not, we assume the cookie is stale. (This may be too restrictive, + * notice case 2.) + * + * 2: when the current contextCSN is known and we have a sync cookie, we search + * for one entry with CSN <= the cookie CSN. (Used to search for =.) If an + * entry is found, the cookie CSN is valid, otherwise it is stale. Case 1 is + * considered a special case of case 2, and both are generally called the + * "find CSN" task. + * + * 3: during a refresh phase, we search for all entries with CSN <= the cookie + * CSN, and generate Present records for them. We always collect this result + * in SyncID sets, even if there's only one match. + */ #define FIND_CSN 1 #define FIND_PRESENT 2 @@ -165,12 +207,16 @@ findcsn_cb( Operation *op, SlapReply *rs slap_callback *sc = op->o_callback; if ( rs->sr_type == REP_SEARCH && rs->sr_err == LDAP_SUCCESS ) { + /* If the private pointer is set, it points to an fcsn_cookie + * and we want to record the maxcsn and match state. + */ if ( sc->sc_private ) { int i; fcsn_cookie *fc = sc->sc_private; + syncrepl_state *srs = op->o_controls[sync_cid]; Attribute *a = attr_find(rs->sr_entry->e_attrs, slap_schema.si_ad_entryCSN ); - i = ber_bvcmp( &a->a_vals[0], op->o_sync_state.ctxcsn ); + i = ber_bvcmp( &a->a_vals[0], srs->sr_state.ctxcsn ); if ( i == 0 ) fc->gotmatch = 1; i = ber_bvcmp( &a->a_vals[0], &fc->maxcsn ); if ( i > 0 ) { @@ -178,12 +224,17 @@ findcsn_cb( Operation *op, SlapReply *rs strcpy(fc->maxcsn.bv_val, a->a_vals[0].bv_val ); } } else { + /* Otherwise, if the private pointer is not set, we just + * want to know if any entry matched the filter. + */ sc->sc_private = (void *)1; } } return LDAP_SUCCESS; } +/* Build a list of entryUUIDs for sending in a SyncID set */ + typedef struct fpres_cookie { int num; BerVarray uuids; @@ -197,7 +248,6 @@ findpres_cb( Operation *op, SlapReply *r int ret = SLAP_CB_CONTINUE; if ( rs->sr_type == REP_SEARCH ) { - Debug(LDAP_DEBUG_TRACE, "present %s\n", rs->sr_entry->e_name.bv_val, 0, 0); ret = slap_build_syncUUID_set( op, &pc->uuids, rs->sr_entry ); if ( ret > 0 ) { pc->num++; @@ -246,13 +296,14 @@ syncprov_findcsn( Operation *op, int mod fcsn_cookie fcookie; fpres_cookie pcookie; int locked = 0; + syncrepl_state *srs = op->o_controls[sync_cid]; - if ( op->o_sync_state.ctxcsn->bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) { + if ( srs->sr_state.ctxcsn->bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) { return LDAP_OTHER; } fop = *op; - fop.o_sync_mode = 0; + fop.o_sync_mode &= SLAP_CONTROL_MASK; /* turn off sync_mode */ fbuf.bv_val = buf; if ( mode == FIND_CSN ) { @@ -270,7 +321,7 @@ syncprov_findcsn( Operation *op, int mod fcookie.maxcsn.bv_val = cbuf; fcookie.maxcsn.bv_len = 0; fcookie.gotmatch = 0; - fbuf.bv_len = sprintf( buf, "(entryCSN>=%s)", op->o_sync_state.ctxcsn->bv_val ); + fbuf.bv_len = sprintf( buf, "(entryCSN>=%s)", srs->sr_state.ctxcsn->bv_val ); } else { if ( locked ) { ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex ); @@ -281,7 +332,7 @@ syncprov_findcsn( Operation *op, int mod fop.ors_attrs = slap_anlist_no_attrs; fop.ors_slimit = 1; cb.sc_private = NULL; - fbuf.bv_len = sprintf( buf, "(entryCSN=%s)", op->o_sync_state.ctxcsn->bv_val ); + fbuf.bv_len = sprintf( buf, "(entryCSN<=%s)", srs->sr_state.ctxcsn->bv_val ); } cb.sc_response = findcsn_cb; @@ -290,15 +341,17 @@ syncprov_findcsn( Operation *op, int mod fop.ors_attrsonly = 0; fop.ors_attrs = uuid_anlist; fop.ors_slimit = SLAP_NO_LIMIT; + /* We want pure entries, not referrals */ + fop.o_managedsait = SLAP_CONTROL_CRITICAL; cb.sc_private = &pcookie; cb.sc_response = findpres_cb; pcookie.num = 0; pcookie.uuids = NULL; - fbuf.bv_len = sprintf( buf, "(entryCSN<=%s)", op->o_sync_state.ctxcsn->bv_val ); + fbuf.bv_len = sprintf( buf, "(entryCSN<=%s)", srs->sr_state.ctxcsn->bv_val ); } cf.f_ava = &eq; cf.f_av_desc = slap_schema.si_ad_entryCSN; - cf.f_av_value = *op->o_sync_state.ctxcsn; + cf.f_av_value = *srs->sr_state.ctxcsn; cf.f_next = NULL; fop.o_callback = &cb; @@ -340,18 +393,23 @@ syncprov_sendresp( Operation *op, opcook Entry e_uuid = {0}; Attribute a_uuid = {0}; Operation sop = *so->s_op; + Opheader ohdr; + syncrepl_state *srs = sop.o_controls[sync_cid]; + ohdr = *sop.o_hdr; + sop.o_hdr = &ohdr; sop.o_tmpmemctx = op->o_tmpmemctx; ctrls[1] = NULL; slap_compose_sync_cookie( op, &cookie, &opc->sctxcsn, - sop.o_sync_state.sid, sop.o_sync_state.rid ); + srs->sr_state.sid, srs->sr_state.rid ); e_uuid.e_attrs = &a_uuid; a_uuid.a_desc = slap_schema.si_ad_entryUUID; a_uuid.a_nvals = &opc->suuid; rs.sr_err = slap_build_sync_state_ctrl( &sop, &rs, &e_uuid, mode, ctrls, 0, 1, &cookie ); + rs.sr_entry = e; rs.sr_ctrls = ctrls; switch( mode ) { @@ -440,8 +498,8 @@ syncprov_matchops( Operation *op, opcook /* validate base */ fc.fss = ss; fc.fbase = 0; - fc.fsuffix = 0; - rc = syncprov_findbase( op, ss, &fc ); + fc.fscope = 0; + rc = syncprov_findbase( op, &fc ); if ( rc != LDAP_SUCCESS ) continue; /* If we're sending results now, look for this op in old matches */ @@ -459,7 +517,7 @@ syncprov_matchops( Operation *op, opcook } /* check if current o_req_dn is in scope and matches filter */ - if ( fc.fsuffix && test_filter( op, e, ss->s_filter ) == + if ( fc.fscope && test_filter( op, e, ss->s_filter ) == LDAP_COMPARE_TRUE ) { if ( saveit ) { sm = op->o_tmpalloc( sizeof(syncmatches), op->o_tmpmemctx ); @@ -511,17 +569,16 @@ syncprov_op_response( Operation *op, Sla { struct berval maxcsn; char cbuf[LDAP_LUTIL_CSNSTR_BUFSIZE]; - void *memctx = op->o_tmpmemctx; cbuf[0] = '\0'; ldap_pvt_thread_mutex_lock( &si->si_csn_mutex ); - op->o_tmpmemctx = NULL; slap_get_commit_csn( op, &maxcsn ); - op->o_tmpmemctx = memctx; if ( maxcsn.bv_val ) { strcpy( cbuf, maxcsn.bv_val ); - free( si->si_ctxcsn.bv_val ); - si->si_ctxcsn = maxcsn; + if ( ber_bvcmp( &maxcsn, &si->si_ctxcsn ) > 0 ) { + strcpy( si->si_ctxcsnbuf, cbuf ); + si->si_ctxcsn.bv_len = maxcsn.bv_len; + } si->si_gotcsn = 1; } ldap_pvt_thread_mutex_unlock( &si->si_csn_mutex ); @@ -660,7 +717,7 @@ syncprov_search_cleanup( Operation *op, op->o_tmpfree( rs->sr_ctrls, op->o_tmpmemctx ); } if ( ss->ss_done ) - op->o_sync_mode = SLAP_SYNC_REFRESH_AND_PERSIST; + op->o_sync_mode |= SLAP_SYNC_REFRESH_AND_PERSIST; return 0; } @@ -670,27 +727,34 @@ syncprov_search_response( Operation *op, searchstate *ss = op->o_callback->sc_private; slap_overinst *on = ss->ss_on; syncprov_info_t *si = on->on_bi.bi_private; + syncrepl_state *srs = op->o_controls[sync_cid]; if ( rs->sr_type == REP_SEARCH || rs->sr_type == REP_SEARCHREF ) { int i; - if ( op->o_sync_state.ctxcsn ) { - Attribute *a = attr_find( rs->sr_entry->e_attrs, - slap_schema.si_ad_entryCSN ); + struct berval cookie; + + Attribute *a = attr_find( rs->sr_entry->e_attrs, + slap_schema.si_ad_entryCSN ); + + if ( srs->sr_state.ctxcsn ) { /* Don't send the ctx entry twice */ - if ( bvmatch( &a->a_nvals[0], op->o_sync_state.ctxcsn )) + if ( bvmatch( &a->a_nvals[0], srs->sr_state.ctxcsn )) return LDAP_SUCCESS; } + slap_compose_sync_cookie( op, &cookie, a->a_nvals, + srs->sr_state.sid, srs->sr_state.rid ); + rs->sr_ctrls = op->o_tmpalloc( sizeof(LDAPControl *)*2, op->o_tmpmemctx ); rs->sr_ctrls[1] = NULL; rs->sr_err = slap_build_sync_state_ctrl( op, rs, rs->sr_entry, - LDAP_SYNC_ADD, rs->sr_ctrls, 0, 0, NULL ); + LDAP_SYNC_ADD, rs->sr_ctrls, 0, 1, &cookie ); } else if ( rs->sr_type == REP_RESULT && rs->sr_err == LDAP_SUCCESS ) { struct berval cookie; slap_compose_sync_cookie( op, &cookie, &op->ors_filter->f_and->f_ava->aa_value, - op->o_sync_state.sid, op->o_sync_state.rid ); + srs->sr_state.sid, srs->sr_state.rid ); /* Is this a regular refresh? */ if ( !ss->ss_so ) { @@ -732,16 +796,19 @@ syncprov_op_search( Operation *op, SlapR Filter *fand, *fava; syncops *sop = NULL; searchstate *ss; + syncrepl_state *srs; - if ( !op->o_sync_mode ) return SLAP_CB_CONTINUE; + if ( !(op->o_sync_mode & SLAP_SYNC_REFRESH) ) return SLAP_CB_CONTINUE; if ( op->ors_deref & LDAP_DEREF_SEARCHING ) { send_ldap_error( op, rs, LDAP_PROTOCOL_ERROR, "illegal value for derefAliases" ); return rs->sr_err; } + srs = op->o_controls[sync_cid]; + /* If this is a persistent search, set it up right away */ - if ( op->o_sync_mode == SLAP_SYNC_REFRESH_AND_PERSIST ) { + if ( op->o_sync_mode & SLAP_SYNC_PERSIST ) { syncops so; fbase_cookie fc; opcookie opc; @@ -757,7 +824,7 @@ syncprov_op_search( Operation *op, SlapR opc.son = on; cb = op->o_callback; op->o_callback = ≻ - rs->sr_err = syncprov_findbase( op, &so, &fc ); + rs->sr_err = syncprov_findbase( op, &fc ); op->o_callback = cb; if ( rs->sr_err != LDAP_SUCCESS ) { @@ -774,9 +841,9 @@ syncprov_op_search( Operation *op, SlapR /* If we have a cookie, handle the PRESENT lookups */ - if ( op->o_sync_state.ctxcsn ) { + if ( srs->sr_state.ctxcsn ) { /* Is the CSN in a valid format? */ - if ( op->o_sync_state.ctxcsn->bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) { + if ( srs->sr_state.ctxcsn->bv_len >= LDAP_LUTIL_CSNSTR_BUFSIZE ) { send_ldap_error( op, rs, LDAP_OTHER, "invalid sync cookie" ); return rs->sr_err; } @@ -792,15 +859,16 @@ syncprov_op_search( Operation *op, SlapR } else { gotstate = 1; /* If just Refreshing and nothing has changed, shortcut it */ - if ( bvmatch( op->o_sync_state.ctxcsn, &si->si_ctxcsn )) { + if ( bvmatch( srs->sr_state.ctxcsn, &si->si_ctxcsn )) { nochange = 1; - if ( op->o_sync_mode == SLAP_SYNC_REFRESH ) { + if ( !(op->o_sync_mode & SLAP_SYNC_PERSIST) ) { LDAPControl *ctrls[2]; ctrls[0] = NULL; ctrls[1] = NULL; slap_build_sync_done_ctrl( op, rs, ctrls, 0, 0, NULL, LDAP_SYNC_REFRESH_DELETES ); + rs->sr_ctrls = ctrls; rs->sr_err = LDAP_SUCCESS; send_ldap_result( op, rs ); return rs->sr_err; @@ -815,16 +883,21 @@ syncprov_op_search( Operation *op, SlapR } } + /* If we didn't get a cookie and we don't know our contextcsn, try to + * find it anyway. + */ if ( !gotstate && !si->si_gotcsn ) { struct berval bv = BER_BVC("1"), *old; - old = op->o_sync_state.ctxcsn; - op->o_sync_state.ctxcsn = &bv; + old = srs->sr_state.ctxcsn; + srs->sr_state.ctxcsn = &bv; syncprov_findcsn( op, FIND_CSN ); - op->o_sync_state.ctxcsn = old; + srs->sr_state.ctxcsn = old; } - /* Append CSN range to search filter */ + /* Append CSN range to search filter, save original filter + * for persistent search evaluation + */ if ( sop ) { sop->s_filter = op->ors_filter; } @@ -844,7 +917,7 @@ syncprov_op_search( Operation *op, SlapR fava->f_choice = LDAP_FILTER_GE; fava->f_ava = op->o_tmpalloc( sizeof(AttributeAssertion), op->o_tmpmemctx ); fava->f_ava->aa_desc = slap_schema.si_ad_entryCSN; - ber_dupbv_x( &fava->f_ava->aa_value, op->o_sync_state.ctxcsn, op->o_tmpmemctx ); + ber_dupbv_x( &fava->f_ava->aa_value, srs->sr_state.ctxcsn, op->o_tmpmemctx ); } fava->f_next = op->ors_filter; op->ors_filter = fand; @@ -863,7 +936,11 @@ shortcut: cb->sc_next = op->o_callback; op->o_callback = cb; - op->o_sync_mode = 0; /* Don't let back-bdb see this */ + /* FIXME: temporary hack to make sure back-bdb's native Psearch handling + * doesn't get invoked. We can skip this after the back-bdb code is + * removed, and also delete ss->ss_done. + */ + op->o_sync_mode &= SLAP_CONTROL_MASK; /* If this is a persistent search and no changes were reported during * the refresh phase, just invoke the response callback to transition @@ -878,80 +955,6 @@ shortcut: return SLAP_CB_CONTINUE; } -#if 0 -static int -syncprov_response( Operation *op, SlapReply *rs ) -{ - slap_overinst *on = (slap_overinst *)op->o_bd->bd_info; - syncprov_info_t *si = (syncprov_info_t *)on->on_bi.bi_private; - - if ( rs->sr_err == LDAP_SUCCESS ) { - if ( op->o_tag == LDAP_REQ_SEARCH ) { - /* handle transition from refresh to persist */ - if ( op->o_sync_mode == SLAP_SYNC_REFRESH_AND_PERSIST ) { - } - - /* If we're checkpointing */ - } else if ( si->si_chkops || si->si_chktime )) { - int do_check = 0; - - switch ( op->o_tag ) { - case LDAP_REQ_EXTENDED: - { int i, doit = 0; - - /* if not PASSWD_MODIFY, break */ - for ( i=0; write_exop[i]; i++ ) - { - if ( !ber_bvcmp( write_exop[i], &op->oq_extended.rs_reqoid )) - { - doit = 1; - break; - } - } - if ( !doit ) break; - } - /* else fallthru */ - case LDAP_REQ_ADD: - case LDAP_REQ_MODIFY: - case LDAP_REQ_MODRDN: - case LDAP_REQ_DELETE: - ldap_pvt_thread_mutex_lock( &si->si_chk_mutex ); - if ( si->si_chkops ) - { - si->si_numops++; - if ( si->si_numops >= si->si_chkops ) - { - do_check = 1; - si->si_numops = 0; - } - } - if ( si->si_chktime ) - { - if ( op->o_time - si->si_chklast >= si->si_chktime ) - { - do_check = 1; - si->si_chklast = op->o_time; - } - } - ldap_pvt_thread_mutex_unlock( &si->si_chk_mutex ); - if ( do_check ) - { - /* write cn=ldapsync to underlying db */ - } - break; - } - } - } - /* Release this DN */ - if ( op->o_tag == LDAP_REQ_MODIFY ) { - ldap_pvt_thread_mutex_lock( &si->si_mod_mutex ); - avl_delete( &si->si_mods, &op->o_req_ndn, dn_avl_cmp ); - ldap_pvt_thread_mutex_unlock( &si->si_mod_mutex ); - } - return SLAP_CB_CONTINUE; -} -#endif - static int syncprov_db_config( BackendDB *be, @@ -994,6 +997,7 @@ syncprov_db_init( on->on_bi.bi_private = si; ldap_pvt_thread_mutex_init( &si->si_csn_mutex ); ldap_pvt_thread_mutex_init( &si->si_ops_mutex ); + si->si_ctxcsn.bv_val = si->si_ctxcsnbuf; csn_anlist[0].an_desc = slap_schema.si_ad_entryCSN; csn_anlist[0].an_name = slap_schema.si_ad_entryCSN->ad_cname; @@ -1001,6 +1005,8 @@ syncprov_db_init( uuid_anlist[0].an_desc = slap_schema.si_ad_entryUUID; uuid_anlist[0].an_name = slap_schema.si_ad_entryUUID->ad_cname; + sync_cid = slap_cids.sc_LDAPsync; + return 0; }