rx: Remove delays in multi_End_Ignore
[openafs.git] / src / ubik / recovery.c
index 89f3843..e622505 100644 (file)
 
 #include <roken.h>
 
-#include <sys/types.h>
-#include <string.h>
-#include <stdarg.h>
-#include <errno.h>
+#include <afs/opr.h>
 
-#ifdef AFS_NT40_ENV
-#include <winsock2.h>
-#include <time.h>
-#include <fcntl.h>
+#ifdef AFS_PTHREAD_ENV
+# include <opr/lock.h>
 #else
-#include <sys/file.h>
-#include <netinet/in.h>
-#include <sys/time.h>
+# include <opr/lockstub.h>
 #endif
 
-#include <lock.h>
-#include <rx/xdr.h>
 #include <rx/rx.h>
 #include <afs/afsutil.h>
+#include <afs/cellconfig.h>
+
 
 #define UBIK_INTERNALS
 #include "ubik.h"
@@ -73,10 +66,6 @@ int
 urecovery_ResetState(void)
 {
     urecovery_state = 0;
-#if !defined(AFS_PTHREAD_ENV)
-    /*  No corresponding LWP_WaitProcess found anywhere for this -- klm */
-    LWP_NoYieldSignal(&urecovery_state);
-#endif
     return 0;
 }
 
@@ -84,17 +73,15 @@ urecovery_ResetState(void)
  * \brief sync site
  *
  * routine called when a non-sync site server goes down; restarts recovery
- * process to send missing server the new db when it comes back up.
+ * process to send missing server the new db when it comes back up for
+ * non-sync site servers.
  *
  * \note This routine should not do anything with variables used by non-sync site servers.
  */
 int
-urecovery_LostServer(void)
+urecovery_LostServer(struct ubik_server *ts)
 {
-#if !defined(AFS_PTHREAD_ENV)
-    /*  No corresponding LWP_WaitProcess found anywhere for this -- klm */
-    LWP_NoYieldSignal(&urecovery_state);
-#endif
+    ubeacon_ReinitServer(ts);
     return 0;
 }
 
@@ -112,7 +99,7 @@ urecovery_AllBetter(struct ubik_dbase *adbase, int areadAny)
 {
     afs_int32 rcode;
 
-    ubik_dprint_25("allbetter checking\n");
+    ViceLog(25, ("allbetter checking\n"));
     rcode = 0;
 
 
@@ -131,11 +118,11 @@ urecovery_AllBetter(struct ubik_dbase *adbase, int areadAny)
      * that the sync site is still the sync site, 'cause it won't talk
      * to us until a timeout period has gone by.  When we recover, we
      * leave this clear until we get a new dbase */
-    else if ((uvote_GetSyncSite() && (vcmp(ubik_dbVersion, ubik_dbase->version) == 0))) {      /* && order is important */
+    else if (uvote_HaveSyncAndVersion(ubik_dbase->version)) {
        rcode = 1;
     }
 
-    ubik_dprint_25("allbetter: returning %d\n", rcode);
+    ViceLog(25, ("allbetter: returning %d\n", rcode));
     return rcode;
 }
 
@@ -146,9 +133,17 @@ int
 urecovery_AbortAll(struct ubik_dbase *adbase)
 {
     struct ubik_trans *tt;
+    int reads = 0, writes = 0;
+
     for (tt = adbase->activeTrans; tt; tt = tt->next) {
+       if (tt->type == UBIK_WRITETRANS)
+           writes++;
+       else
+           reads++;
        udisk_abort(tt);
     }
+    ViceLog(0, ("urecovery_AbortAll: just aborted %d read and %d write transactions\n",
+                   reads, writes));
     return 0;
 }
 
@@ -164,14 +159,22 @@ urecovery_CheckTid(struct ubik_tid *atid, int abortalways)
        if (atid->epoch != ubik_currentTrans->tid.epoch
            || atid->counter > ubik_currentTrans->tid.counter || abortalways) {
            /* don't match, abort it */
+           int endit = 0;
            /* If the thread is not waiting for lock - ok to end it */
-#if !defined(UBIK_PAUSE)
            if (ubik_currentTrans->locktype != LOCKWAIT) {
-#endif /* UBIK_PAUSE */
+               endit = 1;
+           }
+
+           ViceLog(0, ("urecovery_CheckTid: Aborting/ending bad remote "
+                       "transaction. (tx %d.%d, atid %d.%d, abortalways %d, "
+                       "endit %d)\n",
+                       ubik_currentTrans->tid.epoch,
+                       ubik_currentTrans->tid.counter,
+                       atid->epoch, atid->counter,
+                       abortalways, endit));
+           if (endit) {
                udisk_end(ubik_currentTrans);
-#if !defined(UBIK_PAUSE)
            }
-#endif /* UBIK_PAUSE */
            ubik_currentTrans = (struct ubik_trans *)0;
        }
     }
@@ -253,8 +256,8 @@ ReplayLog(struct ubik_dbase *adbase)
            /* otherwise, skip over the data bytes, too */
            tpos += ntohl(buffer[2]) + 3 * sizeof(afs_int32);
        } else {
-           ubik_print("corrupt log opcode (%d) at position %d\n", opcode,
-                      tpos);
+           ViceLog(0, ("corrupt log opcode (%d) at position %d\n", opcode,
+                      tpos));
            break;              /* corrupt log! */
        }
     }
@@ -289,6 +292,9 @@ ReplayLog(struct ubik_dbase *adbase)
                code = (*adbase->setlabel) (adbase, 0, &version);
                if (code)
                    return code;
+               ViceLog(0, ("Successfully replayed log for interrupted "
+                          "transaction; db version is now %ld.%ld\n",
+                          (long) version.epoch, (long) version.counter));
                logIsGood = 1;
                break;          /* all done now */
            } else if (opcode == LOGTRUNCATE) {
@@ -344,8 +350,8 @@ ReplayLog(struct ubik_dbase *adbase)
                    len -= thisSize;
                }
            } else {
-               ubik_print("corrupt log opcode (%d) at position %d\n",
-                          opcode, tpos);
+               ViceLog(0, ("corrupt log opcode (%d) at position %d\n",
+                          opcode, tpos));
                break;          /* corrupt log! */
            }
        }
@@ -355,7 +361,7 @@ ReplayLog(struct ubik_dbase *adbase)
            if (code)
                return code;
        } else {
-           ubik_print("Log read error on pass 2\n");
+           ViceLog(0, ("Log read error on pass 2\n"));
            return UBADLOG;
        }
     }
@@ -378,6 +384,7 @@ InitializeDB(struct ubik_dbase *adbase)
     code = (*adbase->getlabel) (adbase, 0, &adbase->version);
     if (code) {
        /* try setting the label to a new value */
+       UBIK_VERSION_LOCK;
        adbase->version.epoch = 1;      /* value for newly-initialized db */
        adbase->version.counter = 1;
        code = (*adbase->setlabel) (adbase, 0, &adbase->version);
@@ -387,11 +394,7 @@ InitializeDB(struct ubik_dbase *adbase)
            adbase->version.counter = 0;
            (*adbase->setlabel) (adbase, 0, &adbase->version);
        }
-#ifdef AFS_PTHREAD_ENV
-       CV_BROADCAST(&adbase->version_cond);
-#else
-       LWP_NoYieldSignal(&adbase->version);
-#endif
+       UBIK_VERSION_UNLOCK;
     }
     return 0;
 }
@@ -406,10 +409,13 @@ urecovery_Initialize(struct ubik_dbase *adbase)
 {
     afs_int32 code;
 
+    DBHOLD(adbase);
     code = ReplayLog(adbase);
     if (code)
-       return code;
+       goto done;
     code = InitializeDB(adbase);
+done:
+    DBRELE(adbase);
     return code;
 }
 
@@ -447,7 +453,7 @@ urecovery_Initialize(struct ubik_dbase *adbase)
 void *
 urecovery_Interact(void *dummy)
 {
-    afs_int32 code, tcode;
+    afs_int32 code;
     struct ubik_server *bestServer = NULL;
     struct ubik_server *ts;
     int dbok, doingRPC, now;
@@ -462,11 +468,13 @@ urecovery_Interact(void *dummy)
     struct ubik_stat ubikstat;
     struct in_addr inAddr;
     char hoststr[16];
-#ifndef OLD_URECOVERY
     char pbuffer[1028];
     int fd = -1;
     afs_int32 pass;
-#endif
+
+    memset(pbuffer, 0, sizeof(pbuffer));
+
+    opr_threadname_set("recovery");
 
     /* otherwise, begin interaction */
     urecovery_state = 0;
@@ -481,7 +489,7 @@ urecovery_Interact(void *dummy)
        IOMGR_Select(0, 0, 0, 0, &tv);
 #endif
 
-       ubik_dprint("recovery running in state %x\n", urecovery_state);
+       ViceLog(5, ("recovery running in state %x\n", urecovery_state));
 
        /* Every 30 seconds, check all the down servers and mark them
         * as up if they respond. When a server comes up or found to
@@ -490,35 +498,39 @@ urecovery_Interact(void *dummy)
         */
        if ((now = FT_ApproxTime()) > 30 + lastProbeTime) {
 
-#ifdef AFS_PTHREAD_ENV
-           DBHOLD(ubik_dbase);
-#endif
-
            for (ts = ubik_servers, doingRPC = 0; ts; ts = ts->next) {
+               UBIK_BEACON_LOCK;
                if (!ts->up) {
+                   UBIK_BEACON_UNLOCK;
                    doingRPC = 1;
                    code = DoProbe(ts);
                    if (code == 0) {
+                       UBIK_BEACON_LOCK;
                        ts->up = 1;
+                       UBIK_BEACON_UNLOCK;
+                       DBHOLD(ubik_dbase);
                        urecovery_state &= ~UBIK_RECFOUNDDB;
+                       DBRELE(ubik_dbase);
                    }
-               } else if (!ts->currentDB) {
-                   urecovery_state &= ~UBIK_RECFOUNDDB;
+               } else {
+                   UBIK_BEACON_UNLOCK;
+                   DBHOLD(ubik_dbase);
+                   if (!ts->currentDB)
+                       urecovery_state &= ~UBIK_RECFOUNDDB;
+                   DBRELE(ubik_dbase);
                }
            }
 
-#ifdef AFS_PTHREAD_ENV
-           DBRELE(ubik_dbase);
-#endif
-
            if (doingRPC)
                now = FT_ApproxTime();
            lastProbeTime = now;
        }
 
        /* Mark whether we are the sync site */
+       DBHOLD(ubik_dbase);
        if (!ubeacon_AmSyncSite()) {
            urecovery_state &= ~UBIK_RECSYNCSITE;
+           DBRELE(ubik_dbase);
            continue;           /* nothing to do */
        }
        urecovery_state |= UBIK_RECSYNCSITE;
@@ -527,16 +539,25 @@ urecovery_Interact(void *dummy)
         * most current database, then go find the most current db.
         */
        if (!(urecovery_state & UBIK_RECFOUNDDB)) {
+            int okcalls = 0;
+           DBRELE(ubik_dbase);
            bestServer = (struct ubik_server *)0;
            bestDBVersion.epoch = 0;
            bestDBVersion.counter = 0;
            for (ts = ubik_servers; ts; ts = ts->next) {
-               if (!ts->up)
+               UBIK_BEACON_LOCK;
+               if (!ts->up) {
+                   UBIK_BEACON_UNLOCK;
                    continue;   /* don't bother with these guys */
+               }
+               UBIK_BEACON_UNLOCK;
                if (ts->isClone)
                    continue;
+               UBIK_ADDR_LOCK;
                code = DISK_GetVersion(ts->disk_rxcid, &ts->version);
+               UBIK_ADDR_UNLOCK;
                if (code == 0) {
+                    okcalls++;
                    /* perhaps this is the best version */
                    if (vcmp(ts->version, bestDBVersion) > 0) {
                        /* new best version */
@@ -545,34 +566,40 @@ urecovery_Interact(void *dummy)
                    }
                }
            }
-           /* take into consideration our version. Remember if we,
-            * the sync site, have the best version. Also note that
-            * we may need to send the best version out.
-            */
-           if (vcmp(ubik_dbase->version, bestDBVersion) >= 0) {
-               bestDBVersion = ubik_dbase->version;
-               bestServer = (struct ubik_server *)0;
-               urecovery_state |= UBIK_RECHAVEDB;
-           } else {
-               /* Clear the flag only when we know we have to retrieve
-                * the db. Because urecovery_AllBetter() looks at it.
-                */
-               urecovery_state &= ~UBIK_RECHAVEDB;
-           }
-           urecovery_state |= UBIK_RECFOUNDDB;
-           urecovery_state &= ~UBIK_RECSENTDB;
+
+           DBHOLD(ubik_dbase);
+
+            if (okcalls + 1 >= ubik_quorum) {
+                /* If we've asked a majority of sites about their db version,
+                 * then we can say with confidence that we've found the best db
+                 * version. If we haven't contacted most sites (because
+                 * GetVersion failed or because we already know the server is
+                 * down), then we don't really know if we know about the best
+                 * db version. So we can only proceed in here if 'okcalls'
+                 * indicates we managed to contact a majority of sites. */
+
+                /* take into consideration our version. Remember if we,
+                 * the sync site, have the best version. Also note that
+                 * we may need to send the best version out.
+                 */
+                if (vcmp(ubik_dbase->version, bestDBVersion) >= 0) {
+                    bestDBVersion = ubik_dbase->version;
+                    bestServer = (struct ubik_server *)0;
+                    urecovery_state |= UBIK_RECHAVEDB;
+                } else {
+                    /* Clear the flag only when we know we have to retrieve
+                     * the db. Because urecovery_AllBetter() looks at it.
+                     */
+                    urecovery_state &= ~UBIK_RECHAVEDB;
+                }
+                urecovery_state |= UBIK_RECFOUNDDB;
+                urecovery_state &= ~UBIK_RECSENTDB;
+            }
        }
-#if defined(UBIK_PAUSE)
-       /* it's not possible for UBIK_RECFOUNDDB not to be set here.
-        * However, we might have lost UBIK_RECSYNCSITE, and that
-        * IS important.
-        */
-       if (!(urecovery_state & UBIK_RECSYNCSITE))
-           continue;           /* lost sync */
-#else
-       if (!(urecovery_state & UBIK_RECFOUNDDB))
+       if (!(urecovery_state & UBIK_RECFOUNDDB)) {
+           DBRELE(ubik_dbase);
            continue;           /* not ready */
-#endif /* UBIK_PAUSE */
+       }
 
        /* If we, the sync site, do not have the best db version, then
         * go and get it from the server that does.
@@ -581,48 +608,46 @@ urecovery_Interact(void *dummy)
            urecovery_state |= UBIK_RECHAVEDB;
        } else {
            /* we don't have the best version; we should fetch it. */
-           DBHOLD(ubik_dbase);
            urecovery_AbortAll(ubik_dbase);
 
+           pbuffer[0] = '\0';
+
            /* Rx code to do the Bulk fetch */
            file = 0;
            offset = 0;
+           UBIK_ADDR_LOCK;
            rxcall = rx_NewCall(bestServer->disk_rxcid);
 
-           ubik_print("Ubik: Synchronize database with server %s\n",
-                      afs_inet_ntoa_r(bestServer->addr[0], hoststr));
+           ViceLog(0, ("Ubik: Synchronize database: receive (via GetFile) "
+                       "from server %s begin\n",
+                      afs_inet_ntoa_r(bestServer->addr[0], hoststr)));
+           UBIK_ADDR_UNLOCK;
 
            code = StartDISK_GetFile(rxcall, file);
            if (code) {
-               ubik_dprint("StartDiskGetFile failed=%d\n", code);
+               ViceLog(0, ("StartDiskGetFile failed=%d\n", code));
                goto FetchEndCall;
            }
            nbytes = rx_Read(rxcall, (char *)&length, sizeof(afs_int32));
            length = ntohl(length);
            if (nbytes != sizeof(afs_int32)) {
-               ubik_dprint("Rx-read length error=%d\n", code = BULK_ERROR);
+               ViceLog(0, ("Rx-read length error=%d\n", BULK_ERROR));
                code = EIO;
                goto FetchEndCall;
            }
 
-#ifdef OLD_URECOVERY
-           /* Truncate the file first */
-           code = (*ubik_dbase->truncate) (ubik_dbase, file, 0);
-           if (code) {
-               ubik_dprint("truncate io error=%d\n", code);
-               goto FetchEndCall;
-           }
-           tversion.counter = 0;
-#endif
            /* give invalid label during file transit */
+           UBIK_VERSION_LOCK;
            tversion.epoch = 0;
            code = (*ubik_dbase->setlabel) (ubik_dbase, file, &tversion);
+           UBIK_VERSION_UNLOCK;
            if (code) {
-               ubik_dprint("setlabel io error=%d\n", code);
+               ViceLog(0, ("setlabel io error=%d\n", code));
                goto FetchEndCall;
            }
-#ifndef OLD_URECOVERY
-           afs_snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.TMP", ubik_dbase->pathName, (file<0)?"SYS":"", (file<0)?-file:file);
+           snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.TMP",
+                    ubik_dbase->pathName, (file<0)?"SYS":"",
+                    (file<0)?-file:file);
            fd = open(pbuffer, O_CREAT | O_RDWR | O_TRUNC, 0600);
            if (fd < 0) {
                code = errno;
@@ -633,7 +658,6 @@ urecovery_Interact(void *dummy)
                close(fd);
                goto FetchEndCall;
            }
-#endif
 
            pass = 0;
            while (length > 0) {
@@ -644,19 +668,13 @@ urecovery_Interact(void *dummy)
 #endif
                nbytes = rx_Read(rxcall, tbuffer, tlen);
                if (nbytes != tlen) {
-                   ubik_dprint("Rx-read bulk error=%d\n", code = BULK_ERROR);
+                   ViceLog(0, ("Rx-read bulk error=%d\n", BULK_ERROR));
                    code = EIO;
                    close(fd);
                    goto FetchEndCall;
                }
-#ifdef OLD_URECOVERY
-               nbytes =
-                   (*ubik_dbase->write) (ubik_dbase, file, tbuffer, offset,
-                                         tlen);
-#else
                nbytes = write(fd, tbuffer, tlen);
                pass++;
-#endif
                if (nbytes != tlen) {
                    code = UIOERROR;
                    close(fd);
@@ -665,79 +683,80 @@ urecovery_Interact(void *dummy)
                offset += tlen;
                length -= tlen;
            }
-#ifndef OLD_URECOVERY
            code = close(fd);
            if (code)
                goto FetchEndCall;
-#endif
            code = EndDISK_GetFile(rxcall, &tversion);
          FetchEndCall:
-           tcode = rx_EndCall(rxcall, code);
-           if (!code)
-               code = tcode;
+           code = rx_EndCall(rxcall, code);
            if (!code) {
                /* we got a new file, set up its header */
                urecovery_state |= UBIK_RECHAVEDB;
+               UBIK_VERSION_LOCK;
                memcpy(&ubik_dbase->version, &tversion,
                       sizeof(struct ubik_version));
-#ifdef OLD_URECOVERY
-               (*ubik_dbase->sync) (ubik_dbase, 0);    /* get data out first */
-#else
-               afs_snprintf(tbuffer, sizeof(tbuffer), "%s.DB%s%d", ubik_dbase->pathName, (file<0)?"SYS":"", (file<0)?-file:file);
+               snprintf(tbuffer, sizeof(tbuffer), "%s.DB%s%d",
+                        ubik_dbase->pathName, (file<0)?"SYS":"",
+                        (file<0)?-file:file);
 #ifdef AFS_NT40_ENV
-               afs_snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.OLD", ubik_dbase->pathName, (file<0)?"SYS":"", (file<0)?-file:file);
+               snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.OLD",
+                        ubik_dbase->pathName, (file<0)?"SYS":"",
+                        (file<0)?-file:file);
                code = unlink(pbuffer);
                if (!code)
                    code = rename(tbuffer, pbuffer);
-               afs_snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.TMP", ubik_dbase->pathName, (file<0)?"SYS":"", (file<0)?-file:file);
+               snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.TMP",
+                        ubik_dbase->pathName, (file<0)?"SYS":"",
+                        (file<0)?-file:file);
 #endif
                if (!code)
                    code = rename(pbuffer, tbuffer);
                if (!code) {
                    (*ubik_dbase->open) (ubik_dbase, file);
-#endif
                    /* after data is good, sync disk with correct label */
                    code =
                        (*ubik_dbase->setlabel) (ubik_dbase, 0,
                                                 &ubik_dbase->version);
-#ifndef OLD_URECOVERY
                }
+               UBIK_VERSION_UNLOCK;
 #ifdef AFS_NT40_ENV
-               afs_snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.OLD", ubik_dbase->pathName, (file<0)?"SYS":"", (file<0)?-file:file);
+               snprintf(pbuffer, sizeof(pbuffer), "%s.DB%s%d.OLD",
+                        ubik_dbase->pathName, (file<0)?"SYS":"",
+                        (file<0)?-file:file);
                unlink(pbuffer);
 #endif
-#endif
            }
            if (code) {
-#ifndef OLD_URECOVERY
-               unlink(pbuffer);
+               if (pbuffer[0] != '\0') {
+                   unlink(pbuffer);
+               }
                /*
                 * We will effectively invalidate the old data forever now.
                 * Unclear if we *should* but we do.
                 */
-#endif
+               UBIK_VERSION_LOCK;
                ubik_dbase->version.epoch = 0;
                ubik_dbase->version.counter = 0;
-               ubik_print("Ubik: Synchronize database failed (error = %d)\n",
-                          code);
+               UBIK_VERSION_UNLOCK;
+               ViceLog(0,
+                   ("Ubik: Synchronize database: receive (via GetFile) "
+                   "from server %s failed (error = %d)\n",
+                   afs_inet_ntoa_r(bestServer->addr[0], hoststr), code));
            } else {
-               ubik_print("Ubik: Synchronize database completed\n");
+               ViceLog(0,
+                   ("Ubik: Synchronize database: receive (via GetFile) "
+                   "from server %s complete, version: %d.%d\n",
+                   afs_inet_ntoa_r(bestServer->addr[0], hoststr),
+                   ubik_dbase->version.epoch, ubik_dbase->version.counter));
+
                urecovery_state |= UBIK_RECHAVEDB;
            }
            udisk_Invalidate(ubik_dbase, 0);    /* data has changed */
-#ifdef AFS_PTHREAD_ENV
-           CV_BROADCAST(&ubik_dbase->version_cond);
-#else
-           LWP_NoYieldSignal(&ubik_dbase->version);
-#endif
-           DBRELE(ubik_dbase);
        }
-#if defined(UBIK_PAUSE)
-       if (!(urecovery_state & UBIK_RECSYNCSITE))
-           continue;           /* lost sync */
-#endif /* UBIK_PAUSE */
-       if (!(urecovery_state & UBIK_RECHAVEDB))
+       if (!(urecovery_state & UBIK_RECHAVEDB)) {
+           DBRELE(ubik_dbase);
            continue;           /* not ready */
+       }
 
        /* If the database was newly initialized, then when we establish quorum, write
         * a new label. This allows urecovery_AllBetter() to allow access for reads.
@@ -745,20 +764,14 @@ urecovery_Interact(void *dummy)
         * database and overwrite this one.
         */
        if (ubik_dbase->version.epoch == 1) {
-           DBHOLD(ubik_dbase);
            urecovery_AbortAll(ubik_dbase);
-           ubik_epochTime = 2;
-           ubik_dbase->version.epoch = ubik_epochTime;
+           UBIK_VERSION_LOCK;
+           ubik_dbase->version.epoch = 2;
            ubik_dbase->version.counter = 1;
            code =
                (*ubik_dbase->setlabel) (ubik_dbase, 0, &ubik_dbase->version);
+           UBIK_VERSION_UNLOCK;
            udisk_Invalidate(ubik_dbase, 0);    /* data may have changed */
-#ifdef AFS_PTHREAD_ENV
-           CV_BROADCAST(&ubik_dbase->version_cond);
-#else
-           LWP_NoYieldSignal(&ubik_dbase->version);
-#endif
-           DBRELE(ubik_dbase);
        }
 
        /* Check the other sites and send the database to them if they
@@ -768,7 +781,6 @@ urecovery_Interact(void *dummy)
            /* now propagate out new version to everyone else */
            dbok = 1;           /* start off assuming they all worked */
 
-           DBHOLD(ubik_dbase);
            /*
             * Check if a write transaction is in progress. We can't send the
             * db when a write is in progress here because the db would be
@@ -778,50 +790,65 @@ urecovery_Interact(void *dummy)
             * the write-lock above if there is a write transaction in progress,
             * but then, it won't hurt to check, will it?
             */
-           if (ubik_dbase->flags & DBWRITING) {
+           if (ubik_dbase->dbFlags & DBWRITING) {
                struct timeval tv;
                int safety = 0;
-               tv.tv_sec = 0;
-               tv.tv_usec = 50000;
-               while ((ubik_dbase->flags & DBWRITING) && (safety < 500)) {
+               long cur_usec = 50000;
+               while ((ubik_dbase->dbFlags & DBWRITING) && (safety < 500)) {
                    DBRELE(ubik_dbase);
                    /* sleep for a little while */
+                   tv.tv_sec = 0;
+                   tv.tv_usec = cur_usec;
 #ifdef AFS_PTHREAD_ENV
                    select(0, 0, 0, 0, &tv);
 #else
                    IOMGR_Select(0, 0, 0, 0, &tv);
 #endif
-                   tv.tv_usec += 10000;
+                   cur_usec += 10000;
                    safety++;
                    DBHOLD(ubik_dbase);
                }
            }
 
            for (ts = ubik_servers; ts; ts = ts->next) {
+               UBIK_ADDR_LOCK;
                inAddr.s_addr = ts->addr[0];
+               UBIK_ADDR_UNLOCK;
+               UBIK_BEACON_LOCK;
                if (!ts->up) {
-                   ubik_dprint("recovery cannot send version to %s\n",
-                               afs_inet_ntoa_r(inAddr.s_addr, hoststr));
+                   UBIK_BEACON_UNLOCK;
+                   /* It would be nice to have this message at loglevel
+                    * 0 as well, but it will log once every 4s for each
+                    * down server while in this recovery state.  This
+                    * should only be changed to loglevel 0 if it is
+                    * also rate-limited.
+                    */
+                   ViceLog(5, ("recovery cannot send version to %s\n",
+                               afs_inet_ntoa_r(inAddr.s_addr, hoststr)));
                    dbok = 0;
                    continue;
                }
-               ubik_dprint("recovery sending version to %s\n",
-                           afs_inet_ntoa_r(inAddr.s_addr, hoststr));
+               UBIK_BEACON_UNLOCK;
+
                if (vcmp(ts->version, ubik_dbase->version) != 0) {
-                   ubik_dprint("recovery stating local database\n");
+                   ViceLog(0, ("Synchronize database: send (via SendFile) "
+                               "to server %s begin\n",
+                           afs_inet_ntoa_r(inAddr.s_addr, hoststr)));
 
                    /* Rx code to do the Bulk Store */
                    code = (*ubik_dbase->stat) (ubik_dbase, 0, &ubikstat);
                    if (!code) {
                        length = ubikstat.size;
                        file = offset = 0;
+                       UBIK_ADDR_LOCK;
                        rxcall = rx_NewCall(ts->disk_rxcid);
+                       UBIK_ADDR_UNLOCK;
                        code =
                            StartDISK_SendFile(rxcall, file, length,
                                               &ubik_dbase->version);
                        if (code) {
-                           ubik_dprint("StartDiskSendFile failed=%d\n",
-                                       code);
+                           ViceLog(0, ("StartDiskSendFile failed=%d\n",
+                                       code));
                            goto StoreEndCall;
                        }
                        while (length > 0) {
@@ -832,14 +859,14 @@ urecovery_Interact(void *dummy)
                                (*ubik_dbase->read) (ubik_dbase, file,
                                                     tbuffer, offset, tlen);
                            if (nbytes != tlen) {
-                               ubik_dprint("Local disk read error=%d\n",
-                                           code = UIOERROR);
+                               code = UIOERROR;
+                               ViceLog(0, ("Local disk read error=%d\n", code));
                                goto StoreEndCall;
                            }
                            nbytes = rx_Write(rxcall, tbuffer, tlen);
                            if (nbytes != tlen) {
-                               ubik_dprint("Rx-write bulk error=%d\n", code =
-                                           BULK_ERROR);
+                               code = BULK_ERROR;
+                               ViceLog(0, ("Rx-write bulk error=%d\n", code));
                                goto StoreEndCall;
                            }
                            offset += tlen;
@@ -849,23 +876,35 @@ urecovery_Interact(void *dummy)
                      StoreEndCall:
                        code = rx_EndCall(rxcall, code);
                    }
+
                    if (code == 0) {
                        /* we set a new file, process its header */
                        ts->version = ubik_dbase->version;
                        ts->currentDB = 1;
-                   } else
+                       ViceLog(0,
+                           ("Ubik: Synchronize database: send (via SendFile) "
+                           "to server %s complete, version: %d.%d\n",
+                           afs_inet_ntoa_r(inAddr.s_addr, hoststr),
+                           ts->version.epoch, ts->version.counter));
+
+                   } else {
                        dbok = 0;
+                       ViceLog(0,
+                           ("Ubik: Synchronize database: send (via SendFile) "
+                            "to server %s failed (error = %d)\n",
+                           afs_inet_ntoa_r(inAddr.s_addr, hoststr), code));
+                   }
                } else {
                    /* mark file up to date */
                    ts->currentDB = 1;
                }
            }
-           DBRELE(ubik_dbase);
            if (dbok)
                urecovery_state |= UBIK_RECSENTDB;
        }
+       DBRELE(ubik_dbase);
     }
-    return NULL;
+    AFS_UNREACHED(return(NULL));
 }
 
 /*!
@@ -878,18 +917,17 @@ DoProbe(struct ubik_server *server)
 {
     struct rx_connection *conns[UBIK_MAX_INTERFACE_ADDR];
     struct rx_connection *connSuccess = 0;
-    int i, j, success_i = -1;
+    int i, nconns, success_i = -1;
     afs_uint32 addr;
     char buffer[32];
     char hoststr[16];
-    extern afs_int32 ubikSecIndex;
-    extern struct rx_securityClass *ubikSecClass;
 
-    for (i = 0; (addr = server->addr[i]) && (i < UBIK_MAX_INTERFACE_ADDR);
+    UBIK_ADDR_LOCK;
+    for (i = 0; (i < UBIK_MAX_INTERFACE_ADDR) && (addr = server->addr[i]);
         i++) {
        conns[i] =
            rx_NewConnection(addr, ubik_callPortal, DISK_SERVICE_ID,
-                            ubikSecClass, ubikSecIndex);
+                            addr_globals.ubikSecClass, addr_globals.ubikSecIndex);
 
        /* user requirement to use only the primary interface */
        if (ubikPrimaryAddrOnly) {
@@ -897,26 +935,21 @@ DoProbe(struct ubik_server *server)
            break;
        }
     }
-    osi_Assert(i);                     /* at least one interface address for this server */
-
-#ifdef AFS_PTHREAD_ENV
-    DBRELE(ubik_dbase);
-#endif
+    UBIK_ADDR_UNLOCK;
+    nconns = i;
+    opr_Assert(nconns);                        /* at least one interface address for this server */
 
-    multi_Rx(conns, i) {
+    multi_Rx(conns, nconns) {
        multi_DISK_Probe();
        if (!multi_error) {     /* first success */
            success_i = multi_i;
 
            multi_Abort;
        }
-    } multi_End_Ignore;
-
-#ifdef AFS_PTHREAD_ENV
-    DBHOLD(ubik_dbase);
-#endif
+    } multi_End;
 
     if (success_i >= 0) {
+       UBIK_ADDR_LOCK;
        addr = server->addr[success_i]; /* successful interface addr */
 
        if (server->disk_rxcid) /* destroy existing conn */
@@ -927,24 +960,25 @@ DoProbe(struct ubik_server *server)
        /* make new connections */
        server->disk_rxcid = conns[success_i];
        server->vote_rxcid = rx_NewConnection(addr, ubik_callPortal,
-                                             VOTE_SERVICE_ID, ubikSecClass,
-                                             ubikSecIndex);
+                                             VOTE_SERVICE_ID, addr_globals.ubikSecClass,
+                                             addr_globals.ubikSecIndex);
 
        connSuccess = conns[success_i];
        strcpy(buffer, afs_inet_ntoa_r(server->addr[0], hoststr));
 
-       ubik_print("ubik:server %s is back up: will be contacted through %s\n",
-            buffer, afs_inet_ntoa_r(addr, hoststr));
+       ViceLog(0, ("ubik:server %s is back up: will be contacted through %s\n",
+            buffer, afs_inet_ntoa_r(addr, hoststr)));
+       UBIK_ADDR_UNLOCK;
     }
 
     /* Destroy all connections except the one on which we succeeded */
-    for (j = 0; j < i; j++)
-       if (conns[j] != connSuccess)
-           rx_DestroyConnection(conns[j]);
+    for (i = 0; i < nconns; i++)
+       if (conns[i] != connSuccess)
+           rx_DestroyConnection(conns[i]);
 
     if (!connSuccess)
-       ubik_dprint("ubik:server %s still down\n",
-                   afs_inet_ntoa_r(server->addr[0], hoststr));
+       ViceLog(5, ("ubik:server %s still down\n",
+                   afs_inet_ntoa_r(server->addr[0], hoststr)));
 
     if (connSuccess)
        return 0;               /* success */