windows-afsd-findserverbyip-refcount-20081223
[openafs.git] / src / WINNT / afsd / cm_server.c
index b64cab5..7567d61 100644 (file)
 #include <string.h>
 
 #include "afsd.h"
+#include <WINNT\syscfg.h>
+#include <WINNT/afsreg.h>
 #include <osi.h>
 #include <rx/rx.h>
 
 osi_rwlock_t cm_serverLock;
+osi_rwlock_t cm_syscfgLock;
 
 cm_server_t *cm_allServersp;
+afs_uint32   cm_numFileServers = 0;
+afs_uint32   cm_numVldbServers = 0;
 
 void
 cm_ForceNewConnectionsAllServers(void)
@@ -33,7 +38,9 @@ cm_ForceNewConnectionsAllServers(void)
     lock_ObtainRead(&cm_serverLock);
     for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
         cm_GetServerNoLock(tsp);
+        lock_ReleaseRead(&cm_serverLock);
        cm_ForceNewConnections(tsp);
+        lock_ObtainRead(&cm_serverLock);
         cm_PutServerNoLock(tsp);
     }
     lock_ReleaseRead(&cm_serverLock);
@@ -83,21 +90,21 @@ cm_PingServer(cm_server_t *tsp)
                  wasDown ? "down" : "up",
                  tsp->capabilities);
 
+        rxconnp = cm_GetRxConn(connp);
        if (wasDown)
-           rx_SetConnDeadTime(connp->callp, 10);
+           rx_SetConnDeadTime(rxconnp, 10);
        if (tsp->type == CM_SERVER_VLDB) {
-           code = VL_ProbeServer(connp->callp);
+           code = VL_ProbeServer(rxconnp);
        }
        else {
            /* file server */
-           rxconnp = cm_GetRxConn(connp);
            code = RXAFS_GetCapabilities(rxconnp, &caps);
            if (code == RXGEN_OPCODE)
                code = RXAFS_GetTime(rxconnp, &secs, &usecs);
-           rx_PutConnection(rxconnp);
        }
        if (wasDown)
-           rx_SetConnDeadTime(connp->callp, ConnDeadtimeout);
+           rx_SetConnDeadTime(rxconnp, ConnDeadtimeout);
+        rx_PutConnection(rxconnp);
        cm_PutConn(connp);
     }  /* got an unauthenticated connection to this server */
 
@@ -105,6 +112,7 @@ cm_PingServer(cm_server_t *tsp)
     if (code >= 0) {
        /* mark server as up */
        tsp->flags &= ~CM_SERVERFLAG_DOWN;
+        tsp->downTime = 0;
 
        /* we currently handle 32-bits of capabilities */
        if (caps.Capabilities_len > 0) {
@@ -132,8 +140,10 @@ cm_PingServer(cm_server_t *tsp)
                     if (tsrvp->ids[i] != 0) {
                         cm_InitReq(&req);
 
-                        code = cm_GetVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                        lock_ReleaseMutex(&tsp->mx);
+                        code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
                                                 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
+                        lock_ObtainMutex(&tsp->mx);
                         if (code == 0) {
                             cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
                             cm_PutVolume(volp);
@@ -144,10 +154,15 @@ cm_PingServer(cm_server_t *tsp)
         }
     } else {
        /* mark server as down */
-       tsp->flags |= CM_SERVERFLAG_DOWN;
-       if (code != VRESTARTING)
+        if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
+            tsp->flags |= CM_SERVERFLAG_DOWN;
+            tsp->downTime = time(NULL);
+        }
+       if (code != VRESTARTING) {
+            lock_ReleaseMutex(&tsp->mx);
            cm_ForceNewConnections(tsp);
-
+            lock_ObtainMutex(&tsp->mx);
+        }
        osi_Log3(afsd_logp, "cm_PingServer server %s (%s) is down with caps 0x%x",
                  osi_LogSaveString(afsd_logp, hoststr), 
                  tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
@@ -164,8 +179,10 @@ cm_PingServer(cm_server_t *tsp)
                     if (tsrvp->ids[i] != 0) {
                         cm_InitReq(&req);
 
-                        code = cm_GetVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                        lock_ReleaseMutex(&tsp->mx);
+                        code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
                                                 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
+                        lock_ObtainMutex(&tsp->mx);
                         if (code == 0) {
                             cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
                             cm_PutVolume(volp);
@@ -183,8 +200,7 @@ cm_PingServer(cm_server_t *tsp)
     lock_ReleaseMutex(&tsp->mx);
 }
 
-
-void cm_CheckServers(long flags, cm_cell_t *cellp)
+static void cm_CheckServersSingular(afs_uint32 flags, cm_cell_t *cellp)
 {
     /* ping all file servers, up or down, with unauthenticated connection,
      * to find out whether we have all our callbacks from the server still.
@@ -193,17 +209,19 @@ void cm_CheckServers(long flags, cm_cell_t *cellp)
     cm_server_t *tsp;
     int doPing;
     int isDown;
+    int isFS;
 
-    lock_ObtainWrite(&cm_serverLock);
+    lock_ObtainRead(&cm_serverLock);
     for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
         cm_GetServerNoLock(tsp);
-        lock_ReleaseWrite(&cm_serverLock);
+        lock_ReleaseRead(&cm_serverLock);
 
         /* now process the server */
         lock_ObtainMutex(&tsp->mx);
 
         doPing = 0;
         isDown = tsp->flags & CM_SERVERFLAG_DOWN;
+        isFS   = tsp->type == CM_SERVER_FILE;
 
         /* only do the ping if the cell matches the requested cell, or we're
          * matching all cells (cellp == NULL), and if we've requested to ping
@@ -211,7 +229,11 @@ void cm_CheckServers(long flags, cm_cell_t *cellp)
          */
         if ((cellp == NULL || cellp == tsp->cellp) &&
              ((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
-               (!isDown && (flags & CM_FLAG_CHECKUPSERVERS)))) {
+               (!isDown && (flags & CM_FLAG_CHECKUPSERVERS))) &&
+             ((!(flags & CM_FLAG_CHECKVLDBSERVERS) || 
+               !isFS && (flags & CM_FLAG_CHECKVLDBSERVERS)) &&
+              (!(flags & CM_FLAG_CHECKFILESERVERS) || 
+                 isFS && (flags & CM_FLAG_CHECKFILESERVERS)))) {
             doPing = 1;
         }      /* we're supposed to check this up/down server */
         lock_ReleaseMutex(&tsp->mx);
@@ -227,44 +249,547 @@ void cm_CheckServers(long flags, cm_cell_t *cellp)
          */
         cm_GCConnections(tsp);
 
-        lock_ObtainWrite(&cm_serverLock);
+        lock_ObtainRead(&cm_serverLock);
         cm_PutServerNoLock(tsp);
     }
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ReleaseRead(&cm_serverLock);
 }       
 
+static void cm_CheckServersMulti(afs_uint32 flags, cm_cell_t *cellp)
+{
+    /* 
+     * The goal of this function is to probe simultaneously 
+     * probe all of the up/down servers (vldb/file) as 
+     * specified by flags in the minimum number of RPCs.
+     * Effectively that means use one multi_RXAFS_GetCapabilities()
+     * followed by possibly one multi_RXAFS_GetTime() and 
+     * one multi_VL_ProbeServer().
+     *
+     * To make this work we must construct the list of vldb
+     * and file servers that are to be probed as well as the
+     * associated data structures.
+     */
+
+    int srvAddrCount = 0;
+    struct srvAddr **addrs = NULL;
+    cm_conn_t **conns = NULL;
+    struct rx_connection **rxconns = NULL;
+    cm_req_t req;
+    afs_int32 i, j, nconns = 0, maxconns;
+    afs_int32 *conntimer, *results;
+    Capabilities *caps = NULL;
+    cm_server_t ** serversp, *tsp;
+    afs_uint32 isDown, wasDown;
+    afs_uint32 code;
+    time_t start, end, *deltas;
+    afs_int32 secs;
+    afs_int32 usecs;
+    char hoststr[16];
+
+    cm_InitReq(&req);
+    maxconns = max(cm_numFileServers,cm_numVldbServers);
+    if (maxconns == 0)
+        return;
+
+    conns = (cm_conn_t **)malloc(maxconns * sizeof(cm_conn_t *));
+    rxconns = (struct rx_connection **)malloc(maxconns * sizeof(struct rx_connection *));
+    conntimer = (afs_int32 *)malloc(maxconns * sizeof (afs_int32));
+    deltas = (time_t *)malloc(maxconns * sizeof (time_t));
+    results = (afs_int32 *)malloc(maxconns * sizeof (afs_int32));
+    serversp = (cm_server_t **)malloc(maxconns * sizeof(cm_server_t *));
+    caps = (Capabilities *)malloc(maxconns * sizeof(Capabilities));
+
+    memset(caps, 0, maxconns * sizeof(Capabilities));
+
+    if ((flags & CM_FLAG_CHECKFILESERVERS) || 
+        !(flags & (CM_FLAG_CHECKFILESERVERS|CM_FLAG_CHECKVLDBSERVERS)))
+    {
+        lock_ObtainRead(&cm_serverLock);
+        for (nconns=0, tsp = cm_allServersp; tsp && nconns < maxconns; tsp = tsp->allNextp) {
+            if (tsp->type != CM_SERVER_FILE || 
+                tsp->cellp == NULL ||           /* SetPref only */
+                cellp && cellp != tsp->cellp)
+                continue;
+
+            cm_GetServerNoLock(tsp);
+            lock_ReleaseRead(&cm_serverLock);
+
+            lock_ObtainMutex(&tsp->mx);
+            isDown = tsp->flags & CM_SERVERFLAG_DOWN;
+
+            if ((tsp->flags & CM_SERVERFLAG_PINGING) ||
+                !((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
+                   (!isDown && (flags & CM_FLAG_CHECKUPSERVERS)))) {
+                lock_ReleaseMutex(&tsp->mx);
+                lock_ObtainRead(&cm_serverLock);
+                cm_PutServerNoLock(tsp);
+                continue;
+            }
+
+            tsp->flags |= CM_SERVERFLAG_PINGING;
+            lock_ReleaseMutex(&tsp->mx);
+
+            serversp[nconns] = tsp;
+            code = cm_ConnByServer(tsp, cm_rootUserp, &conns[nconns]);
+            if (code) {
+                lock_ObtainRead(&cm_serverLock);
+                cm_PutServerNoLock(tsp);
+                continue;
+            }
+            lock_ObtainRead(&cm_serverLock);
+            rxconns[nconns] = cm_GetRxConn(conns[nconns]);
+            if (conntimer[nconns] = (isDown ? 1 : 0))
+                rx_SetConnDeadTime(rxconns[nconns], 10);
+
+            nconns++;
+        }
+        lock_ReleaseRead(&cm_serverLock);
+
+        if (nconns) {
+            /* Perform the multi call */
+            start = time(NULL);
+            multi_Rx(rxconns,nconns)
+            {
+                multi_RXAFS_GetCapabilities(&caps[multi_i]);
+                results[multi_i]=multi_error;
+            } multi_End;
+        }
+
+        /* Process results of servers that support RXAFS_GetCapabilities */
+        for (i=0; i<nconns; i++) {
+            /* Leave the servers that did not support GetCapabilities alone */
+            if (results[i] == RXGEN_OPCODE)
+                continue;
+
+            if (conntimer[i])
+                rx_SetConnDeadTime(rxconns[i], ConnDeadtimeout);
+            rx_PutConnection(rxconns[i]);
+            cm_PutConn(conns[i]);
+
+            tsp = serversp[i];
+            cm_GCConnections(tsp);
+
+            lock_ObtainMutex(&tsp->mx);
+            wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
+
+            if (results[i] >= 0)  {
+                /* mark server as up */
+                tsp->flags &= ~CM_SERVERFLAG_DOWN;
+                tsp->downTime = 0;
+
+                /* we currently handle 32-bits of capabilities */
+                if (caps[i].Capabilities_len > 0) {
+                    tsp->capabilities = caps[i].Capabilities_val[0];
+                    free(caps[i].Capabilities_val);
+                    caps[i].Capabilities_len = 0;
+                    caps[i].Capabilities_val = 0;
+                } else {
+                    tsp->capabilities = 0;
+                }
+
+                afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
+                osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is up with caps 0x%x",
+                          osi_LogSaveString(afsd_logp, hoststr), 
+                          tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
+                          tsp->capabilities);
+
+                /* Now update the volume status if necessary */
+                if (wasDown) {
+                    cm_server_vols_t * tsrvp;
+                    cm_volume_t * volp;
+                    int i;
+
+                    for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
+                        for (i=0; i<NUM_SERVER_VOLS; i++) {
+                            if (tsrvp->ids[i] != 0) {
+                                cm_InitReq(&req);
+
+                                lock_ReleaseMutex(&tsp->mx);
+                                code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                                                         &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
+                                lock_ObtainMutex(&tsp->mx);
+                                if (code == 0) {
+                                    cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
+                                    cm_PutVolume(volp);
+                                }
+                            }
+                        }
+                    }
+                }
+            } else {
+                /* mark server as down */
+                if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
+                    tsp->flags |= CM_SERVERFLAG_DOWN;
+                    tsp->downTime = time(NULL);
+                }
+                if (code != VRESTARTING) {
+                    lock_ReleaseMutex(&tsp->mx);
+                    cm_ForceNewConnections(tsp);
+                    lock_ObtainMutex(&tsp->mx);
+                }
+                afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
+                osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is down with caps 0x%x",
+                          osi_LogSaveString(afsd_logp, hoststr), 
+                          tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
+                          tsp->capabilities);
+
+                /* Now update the volume status if necessary */
+                if (!wasDown) {
+                    cm_server_vols_t * tsrvp;
+                    cm_volume_t * volp;
+                    int i;
+
+                    for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
+                        for (i=0; i<NUM_SERVER_VOLS; i++) {
+                            if (tsrvp->ids[i] != 0) {
+                                cm_InitReq(&req);
+
+                                lock_ReleaseMutex(&tsp->mx);
+                                code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                                                         &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
+                                lock_ObtainMutex(&tsp->mx);
+                                if (code == 0) {
+                                    cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
+                                    cm_PutVolume(volp);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (tsp->waitCount == 0)
+                tsp->flags &= ~CM_SERVERFLAG_PINGING;
+            else 
+                osi_Wakeup((LONG_PTR)tsp);
+            
+            lock_ReleaseMutex(&tsp->mx);
+
+            cm_PutServer(tsp);
+        }
+
+        /* 
+         * At this point we have handled any responses that did not indicate
+         * that RXAFS_GetCapabilities is not supported.
+         */
+        for ( i=0, j=0; i<nconns; i++) {
+            if (results[i] == RXGEN_OPCODE) {
+                if (i != j) {
+                    conns[j] = conns[i];
+                    rxconns[j] = rxconns[i];
+                    serversp[j] = serversp[i];
+                }
+                j++;
+            }
+        }
+        nconns = j;
+
+        if (nconns) {
+            /* Perform the multi call */
+            start = time(NULL);
+            multi_Rx(rxconns,nconns)
+            {
+                secs = usecs = 0;
+                multi_RXAFS_GetTime(&secs, &usecs);
+                end = time(NULL);
+                results[multi_i]=multi_error;
+                if ((start == end) && !multi_error)
+                    deltas[multi_i] = end - secs;
+            } multi_End;
+        }
+
+        /* Process Results of servers that only support RXAFS_GetTime */
+        for (i=0; i<nconns; i++) {
+            /* Leave the servers that did not support GetCapabilities alone */
+            if (conntimer[i])
+                rx_SetConnDeadTime(rxconns[i], ConnDeadtimeout);
+            rx_PutConnection(rxconns[i]);
+            cm_PutConn(conns[i]);
+
+            tsp = serversp[i];
+            cm_GCConnections(tsp);
+
+            lock_ObtainMutex(&tsp->mx);
+            wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
+
+            if (results[i] >= 0)  {
+                /* mark server as up */
+                tsp->flags &= ~CM_SERVERFLAG_DOWN;
+                tsp->downTime = 0;
+                tsp->capabilities = 0;
+
+                afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
+                osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is up with caps 0x%x",
+                          osi_LogSaveString(afsd_logp, hoststr), 
+                          tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
+                          tsp->capabilities);
+
+                /* Now update the volume status if necessary */
+                if (wasDown) {
+                    cm_server_vols_t * tsrvp;
+                    cm_volume_t * volp;
+                    int i;
+
+                    for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
+                        for (i=0; i<NUM_SERVER_VOLS; i++) {
+                            if (tsrvp->ids[i] != 0) {
+                                cm_InitReq(&req);
+
+                                lock_ReleaseMutex(&tsp->mx);
+                                code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                                                         &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
+                                lock_ObtainMutex(&tsp->mx);
+                                if (code == 0) {
+                                    cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
+                                    cm_PutVolume(volp);
+                                }
+                            }
+                        }
+                    }
+                }
+            } else {
+                /* mark server as down */
+                if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
+                    tsp->flags |= CM_SERVERFLAG_DOWN;
+                    tsp->downTime = time(NULL);
+                }
+                if (code != VRESTARTING) {
+                    lock_ReleaseMutex(&tsp->mx);
+                    cm_ForceNewConnections(tsp);
+                    lock_ObtainMutex(&tsp->mx);
+                }
+                afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
+                osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is down with caps 0x%x",
+                          osi_LogSaveString(afsd_logp, hoststr), 
+                          tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
+                          tsp->capabilities);
+
+                /* Now update the volume status if necessary */
+                if (!wasDown) {
+                    cm_server_vols_t * tsrvp;
+                    cm_volume_t * volp;
+                    int i;
+
+                    for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
+                        for (i=0; i<NUM_SERVER_VOLS; i++) {
+                            if (tsrvp->ids[i] != 0) {
+                                cm_InitReq(&req);
+
+                                lock_ReleaseMutex(&tsp->mx);
+                                code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                                                         &req, CM_GETVOL_FLAG_NO_LRU_UPDATE, &volp);
+                                lock_ObtainMutex(&tsp->mx);
+                                if (code == 0) {
+                                    cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
+                                    cm_PutVolume(volp);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (tsp->waitCount == 0)
+                tsp->flags &= ~CM_SERVERFLAG_PINGING;
+            else 
+                osi_Wakeup((LONG_PTR)tsp);
+            
+            lock_ReleaseMutex(&tsp->mx);
+
+            cm_PutServer(tsp);
+        }
+    }
+
+    if ((flags & CM_FLAG_CHECKVLDBSERVERS) || 
+        !(flags & (CM_FLAG_CHECKFILESERVERS|CM_FLAG_CHECKVLDBSERVERS)))
+    {
+        lock_ObtainRead(&cm_serverLock);
+        for (nconns=0, tsp = cm_allServersp; tsp && nconns < maxconns; tsp = tsp->allNextp) {
+            if (tsp->type != CM_SERVER_VLDB ||
+                tsp->cellp == NULL ||           /* SetPref only */
+                cellp && cellp != tsp->cellp)
+                continue;
+
+            cm_GetServerNoLock(tsp);
+            lock_ReleaseRead(&cm_serverLock);
+
+            lock_ObtainMutex(&tsp->mx);
+            isDown = tsp->flags & CM_SERVERFLAG_DOWN;
+
+            if ((tsp->flags & CM_SERVERFLAG_PINGING) ||
+                !((isDown && (flags & CM_FLAG_CHECKDOWNSERVERS)) ||
+                   (!isDown && (flags & CM_FLAG_CHECKUPSERVERS)))) {
+                lock_ReleaseMutex(&tsp->mx);
+                lock_ObtainRead(&cm_serverLock);
+                cm_PutServerNoLock(tsp);
+                continue;
+            }
+
+            tsp->flags |= CM_SERVERFLAG_PINGING;
+            lock_ReleaseMutex(&tsp->mx);
+
+            serversp[nconns] = tsp;
+            code = cm_ConnByServer(tsp, cm_rootUserp, &conns[nconns]);
+            if (code) {
+                lock_ObtainRead(&cm_serverLock);
+                cm_PutServerNoLock(tsp);
+                continue;
+            }
+            lock_ObtainRead(&cm_serverLock);
+            rxconns[nconns] = cm_GetRxConn(conns[nconns]);
+            conntimer[nconns] = (isDown ? 1 : 0);
+            if (isDown)
+                rx_SetConnDeadTime(rxconns[nconns], 10);
+
+            nconns++;
+        }
+        lock_ReleaseRead(&cm_serverLock);
+
+        if (nconns) {
+            /* Perform the multi call */
+            start = time(NULL);
+            multi_Rx(rxconns,nconns)
+            {
+                multi_VL_ProbeServer();
+                results[multi_i]=multi_error;
+            } multi_End;
+        }
+
+        /* Process results of servers that support VL_ProbeServer */
+        for (i=0; i<nconns; i++) {
+            if (conntimer[i])
+                rx_SetConnDeadTime(rxconns[i], ConnDeadtimeout);
+            rx_PutConnection(rxconns[i]);
+            cm_PutConn(conns[i]);
+
+            tsp = serversp[i];
+            cm_GCConnections(tsp);
+
+            lock_ObtainMutex(&tsp->mx);
+            wasDown = tsp->flags & CM_SERVERFLAG_DOWN;
+
+            if (results[i] >= 0)  {
+                /* mark server as up */
+                tsp->flags &= ~CM_SERVERFLAG_DOWN;
+                tsp->downTime = 0;
+                tsp->capabilities = 0;
+
+                afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
+                osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is up with caps 0x%x",
+                          osi_LogSaveString(afsd_logp, hoststr), 
+                          tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
+                          tsp->capabilities);
+            } else {
+                /* mark server as down */
+                if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
+                    tsp->flags |= CM_SERVERFLAG_DOWN;
+                    tsp->downTime = time(NULL);
+                }
+                if (code != VRESTARTING) {
+                    lock_ReleaseMutex(&tsp->mx);
+                    cm_ForceNewConnections(tsp);
+                    lock_ObtainMutex(&tsp->mx);
+                }
+                afs_inet_ntoa_r(tsp->addr.sin_addr.S_un.S_addr, hoststr);
+                osi_Log3(afsd_logp, "cm_MultiPingServer server %s (%s) is down with caps 0x%x",
+                          osi_LogSaveString(afsd_logp, hoststr), 
+                          tsp->type == CM_SERVER_VLDB ? "vldb" : "file",
+                          tsp->capabilities);
+            }
+
+            if (tsp->waitCount == 0)
+                tsp->flags &= ~CM_SERVERFLAG_PINGING;
+            else 
+                osi_Wakeup((LONG_PTR)tsp);
+            
+            lock_ReleaseMutex(&tsp->mx);
+
+            cm_PutServer(tsp);
+        }
+    }
+
+    free(conns);
+    free(rxconns);
+    free(conntimer);
+    free(deltas);
+    free(results);
+    free(serversp);
+    free(caps);
+}
+
+void cm_CheckServers(afs_uint32 flags, cm_cell_t *cellp)
+{
+    DWORD code;
+    HKEY parmKey;
+    DWORD dummyLen;
+    DWORD multi = 1;
+
+    code = RegOpenKeyEx(HKEY_LOCAL_MACHINE, AFSREG_CLT_SVC_PARAM_SUBKEY,
+                         0, KEY_QUERY_VALUE, &parmKey);
+    if (code == ERROR_SUCCESS) {
+        dummyLen = sizeof(multi);
+        code = RegQueryValueEx(parmKey, "MultiCheckServers", NULL, NULL,
+                                (BYTE *) &multi, &dummyLen);
+        RegCloseKey (parmKey);
+    }
+
+    if (multi)
+        cm_CheckServersMulti(flags, cellp);
+    else
+        cm_CheckServersSingular(flags, cellp);
+}
+
 void cm_InitServer(void)
 {
     static osi_once_t once;
         
     if (osi_Once(&once)) {
-        lock_InitializeRWLock(&cm_serverLock, "cm_serverLock");
+        lock_InitializeRWLock(&cm_serverLock, "cm_serverLock", LOCK_HIERARCHY_SERVER_GLOBAL);
+        lock_InitializeRWLock(&cm_syscfgLock, "cm_syscfgLock", LOCK_HIERARCHY_SYSCFG_GLOBAL);
         osi_EndOnce(&once);
     }
 }
 
+/* Protected by cm_syscfgLock (rw) */
+int cm_noIPAddr;         /* number of client network interfaces */
+int cm_IPAddr[CM_MAXINTERFACE_ADDR];    /* client's IP address in host order */
+int cm_SubnetMask[CM_MAXINTERFACE_ADDR];/* client's subnet mask in host order*/
+int cm_NetMtu[CM_MAXINTERFACE_ADDR];    /* client's MTU sizes */
+int cm_NetFlags[CM_MAXINTERFACE_ADDR];  /* network flags */
+int cm_LanAdapterChangeDetected = 1;
+
+void cm_SetLanAdapterChangeDetected(void)
+{
+    lock_ObtainWrite(&cm_syscfgLock);
+    cm_LanAdapterChangeDetected = 1;
+    lock_ReleaseWrite(&cm_syscfgLock);
+}
+
 void cm_GetServer(cm_server_t *serverp)
 {
-    lock_ObtainWrite(&cm_serverLock);
-    serverp->refCount++;
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ObtainRead(&cm_serverLock);
+    InterlockedIncrement(&serverp->refCount);
+    lock_ReleaseRead(&cm_serverLock);
 }
 
 void cm_GetServerNoLock(cm_server_t *serverp)
 {
-    serverp->refCount++;
+    InterlockedIncrement(&serverp->refCount);
 }
 
 void cm_PutServer(cm_server_t *serverp)
 {
-    lock_ObtainWrite(&cm_serverLock);
-    osi_assert(serverp->refCount-- > 0);
-    lock_ReleaseWrite(&cm_serverLock);
+    afs_int32 refCount;
+    lock_ObtainRead(&cm_serverLock);
+    refCount = InterlockedDecrement(&serverp->refCount);
+    osi_assertx(refCount >= 0, "cm_server_t refCount underflow");
+    lock_ReleaseRead(&cm_serverLock);
 }
 
 void cm_PutServerNoLock(cm_server_t *serverp)
 {
-    osi_assert(serverp->refCount-- > 0);
+    afs_int32 refCount = InterlockedDecrement(&serverp->refCount);
+    osi_assertx(refCount >= 0, "cm_server_t refCount underflow");
 }
 
 void cm_SetServerNo64Bit(cm_server_t * serverp, int no64bit)
@@ -293,19 +818,23 @@ void cm_SetServerPrefs(cm_server_t * serverp)
     unsigned long      myAddr, myNet, mySubnet;/* in host byte order */
     unsigned long      netMask;
     int                i;
-
-    int cm_noIPAddr;         /* number of client network interfaces */
-    int cm_IPAddr[CM_MAXINTERFACE_ADDR];    /* client's IP address in host order */
-    int cm_SubnetMask[CM_MAXINTERFACE_ADDR];/* client's subnet mask in host order*/
-    int cm_NetMtu[CM_MAXINTERFACE_ADDR];    /* client's MTU sizes */
-    int cm_NetFlags[CM_MAXINTERFACE_ADDR];  /* network flags */
     long code;
-
-    /* get network related info */
-    cm_noIPAddr = CM_MAXINTERFACE_ADDR;
-    code = syscfg_GetIFInfo(&cm_noIPAddr,
-                           cm_IPAddr, cm_SubnetMask,
-                           cm_NetMtu, cm_NetFlags);
+    int writeLock = 0;
+
+    lock_ObtainRead(&cm_syscfgLock);
+    if (cm_LanAdapterChangeDetected) {
+        lock_ConvertRToW(&cm_syscfgLock);
+        writeLock = 1;
+        if (cm_LanAdapterChangeDetected) {
+            /* get network related info */
+            cm_noIPAddr = CM_MAXINTERFACE_ADDR;
+            code = syscfg_GetIFInfo(&cm_noIPAddr,
+                                     cm_IPAddr, cm_SubnetMask,
+                                     cm_NetMtu, cm_NetFlags);
+            cm_LanAdapterChangeDetected = 0;
+        }
+        lock_ConvertWToR(&cm_syscfgLock);
+    }
 
     serverAddr = ntohl(serverp->addr.sin_addr.s_addr);
     serverp->ipRank  = CM_IPRANK_LOW;  /* default setings */
@@ -344,12 +873,13 @@ void cm_SetServerPrefs(cm_server_t * serverp)
        /* random between 0..15*/
        serverp->ipRank += min(serverp->ipRank, rand() % 0x000f);
     } /* and of for loop */
+    lock_ReleaseRead(&cm_syscfgLock);
 }
 
-cm_server_t *cm_NewServer(struct sockaddr_in *socketp, int type, cm_cell_t *cellp) {
+cm_server_t *cm_NewServer(struct sockaddr_in *socketp, int type, cm_cell_t *cellp, afs_uint32 flags) {
     cm_server_t *tsp;
 
-    osi_assert(socketp->sin_family == AF_INET);
+    osi_assertx(socketp->sin_family == AF_INET, "unexpected socket family");
 
     tsp = malloc(sizeof(*tsp));
     if (tsp) {
@@ -357,30 +887,63 @@ cm_server_t *cm_NewServer(struct sockaddr_in *socketp, int type, cm_cell_t *cell
         tsp->type = type;
         tsp->cellp = cellp;
         tsp->refCount = 1;
-        lock_InitializeMutex(&tsp->mx, "cm_server_t mutex");
+        lock_InitializeMutex(&tsp->mx, "cm_server_t mutex", LOCK_HIERARCHY_SERVER);
         tsp->addr = *socketp;
-        tsp->flags = CM_SERVERFLAG_DOWN;       /* assume down; ping will mark up if available */
 
         cm_SetServerPrefs(tsp); 
 
         lock_ObtainWrite(&cm_serverLock);      /* get server lock */
         tsp->allNextp = cm_allServersp;
         cm_allServersp = tsp;
+
+        switch (type) {
+        case CM_SERVER_VLDB:
+            cm_numVldbServers++;
+            break;      
+        case CM_SERVER_FILE:
+            cm_numFileServers++;
+            break;
+        }
+
         lock_ReleaseWrite(&cm_serverLock);     /* release server lock */
 
-        cm_PingServer(tsp);                    /* Obtain Capabilities and check up/down state */
+        if ( !(flags & CM_FLAG_NOPROBE) ) {
+            tsp->flags = CM_SERVERFLAG_DOWN;   /* assume down; ping will mark up if available */
+            cm_PingServer(tsp);                        /* Obtain Capabilities and check up/down state */
+        }
     }
     return tsp;
 }
 
+cm_server_t *
+cm_FindServerByIP(afs_uint32 ipaddr, int type)
+{
+    cm_server_t *tsp;
+
+    lock_ObtainRead(&cm_serverLock);
+    for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
+        if (tsp->type == type &&
+            tsp->addr.sin_addr.S_un.S_addr == ipaddr)
+            break;
+    }
+
+    /* bump ref count if we found the server */
+    if (tsp) 
+        cm_GetServerNoLock(tsp);
+
+    lock_ReleaseRead(&cm_serverLock);
+
+    return tsp;
+}
+
 /* find a server based on its properties */
 cm_server_t *cm_FindServer(struct sockaddr_in *addrp, int type)
 {
     cm_server_t *tsp;
 
-    osi_assert(addrp->sin_family == AF_INET);
+    osi_assertx(addrp->sin_family == AF_INET, "unexpected socket value");
         
-    lock_ObtainWrite(&cm_serverLock);
+    lock_ObtainRead(&cm_serverLock);
     for (tsp = cm_allServersp; tsp; tsp=tsp->allNextp) {
         if (tsp->type == type &&
             tsp->addr.sin_addr.s_addr == addrp->sin_addr.s_addr) 
@@ -392,7 +955,7 @@ cm_server_t *cm_FindServer(struct sockaddr_in *addrp, int type)
         cm_GetServerNoLock(tsp);
 
     /* drop big table lock */
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ReleaseRead(&cm_serverLock);
        
     /* return what we found */
     return tsp;
@@ -474,8 +1037,10 @@ LONG_PTR cm_ChecksumServerList(cm_serverRef_t *serversp)
     int first = 1;
     cm_serverRef_t *tsrp;
 
-    lock_ObtainWrite(&cm_serverLock);
+    lock_ObtainRead(&cm_serverLock);
     for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
+        if (tsrp->status == srv_deleted)
+            continue;
         if (first)
             first = 0;
         else
@@ -483,13 +1048,13 @@ LONG_PTR cm_ChecksumServerList(cm_serverRef_t *serversp)
         sum ^= (LONG_PTR) tsrp->server;
     }
 
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ReleaseRead(&cm_serverLock);
     return sum;
 }
 
 /*
 ** Insert a server into the server list keeping the list sorted in 
-** asending order of ipRank. 
+** ascending order of ipRank. 
 ** 
 ** The refCount of the cm_serverRef_t is increased
 */
@@ -615,17 +1180,42 @@ void cm_RandomizeServer(cm_serverRef_t** list)
 void cm_FreeServer(cm_server_t* serverp)
 {
     cm_server_vols_t * tsrvp, *nextp;
+    int delserver = 0;
 
     cm_PutServerNoLock(serverp);
     if (serverp->refCount == 0)
     {
-        /* we need to check to ensure that all of the connections
+        /* 
+         * we need to check to ensure that all of the connections
          * for this server have a 0 refCount; otherwise, they will
          * not be garbage collected 
+         *
+         * must drop the cm_serverLock because cm_GCConnections
+         * obtains the cm_connLock and that comes first in the 
+         * lock hierarchy.  
          */
+        lock_ReleaseWrite(&cm_serverLock);
         cm_GCConnections(serverp);  /* connsp */
+        lock_ObtainWrite(&cm_serverLock);
+    }
+
 
+    /* 
+     * Once we have the cm_serverLock locked check to make
+     * sure the refCount is still zero before removing the 
+     * server entirely.
+     */
+    if (serverp->refCount == 0) {
        if (!(serverp->flags & CM_SERVERFLAG_PREF_SET)) {
+            switch (serverp->type) {
+            case CM_SERVER_VLDB:
+                cm_numVldbServers--;
+                break;      
+            case CM_SERVER_FILE:
+                cm_numFileServers--;
+                break;
+            }
+
            lock_FinalizeMutex(&serverp->mx);
            if ( cm_allServersp == serverp )
                cm_allServersp = serverp->allNextp;
@@ -651,6 +1241,7 @@ void cm_FreeServer(cm_server_t* serverp)
     }
 }
 
+/* Called with cm_serverLock write locked */
 void cm_RemoveVolumeFromServer(cm_server_t * serverp, afs_uint32 volID)
 {
     cm_server_vols_t * tsrvp;
@@ -675,6 +1266,9 @@ void cm_FreeServerList(cm_serverRef_t** list, afs_uint32 flags)
     cm_serverRef_t  **nextp = 0;
     cm_serverRef_t  * next = 0;
 
+       if (*list == NULL)
+               return;
+
     lock_ObtainWrite(&cm_serverLock);
 
     while (*current)
@@ -700,4 +1294,3 @@ void cm_FreeServerList(cm_serverRef_t** list, afs_uint32 flags)
   
     lock_ReleaseWrite(&cm_serverLock);
 }
-