Windows: Protect against infinite VIO retries
[openafs.git] / src / WINNT / afsd / cm_conn.c
index 21044e4..eb8fd3b 100644 (file)
@@ -211,7 +211,8 @@ void cm_InitReq(cm_req_t *reqp)
 }
 
 long cm_GetVolServerList(cm_volume_t *volp, afs_uint32 volid, struct cm_user *userp,
-       struct cm_req *reqp, afs_uint32 *replicated, cm_serverRef_t ***serversppp)
+                         struct cm_req *reqp, afs_uint32 *replicated,
+                         cm_serverRef_t ***serversppp)
 {
     *serversppp = cm_GetVolServers(volp, volid, userp, reqp, replicated);
     return (*serversppp ? 0 : CM_ERROR_NOSUCHVOLUME);
@@ -245,13 +246,13 @@ long cm_GetServerList(struct cm_fid *fidp, struct cm_user *userp,
     return (*serversppp ? 0 : CM_ERROR_NOSUCHVOLUME);
 }
 
-void
-cm_SetServerBusyStatus(cm_serverRef_t *serversp, cm_server_t *serverp)
+static void
+cm_SetServerBusyStatus(cm_serverRef_t **serverspp, cm_server_t *serverp)
 {
     cm_serverRef_t *tsrp;
 
-    lock_ObtainWrite(&cm_serverLock);
-    for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
+    lock_ObtainRead(&cm_serverLock);
+    for (tsrp = *serverspp; tsrp; tsrp=tsrp->next) {
         if (tsrp->status == srv_deleted)
             continue;
         if (cm_ServerEqual(tsrp->server, serverp) && tsrp->status == srv_not_busy) {
@@ -259,23 +260,23 @@ cm_SetServerBusyStatus(cm_serverRef_t *serversp, cm_server_t *serverp)
             break;
         }
     }
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ReleaseRead(&cm_serverLock);
 }
 
-void
-cm_ResetServerBusyStatus(cm_serverRef_t *serversp)
+static void
+cm_ResetServerBusyStatus(cm_serverRef_t **serverspp)
 {
     cm_serverRef_t *tsrp;
 
-    lock_ObtainWrite(&cm_serverLock);
-    for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
+    lock_ObtainRead(&cm_serverLock);
+    for (tsrp = *serverspp; tsrp; tsrp=tsrp->next) {
         if (tsrp->status == srv_deleted)
             continue;
         if (tsrp->status == srv_busy) {
             tsrp->status = srv_not_busy;
         }
     }
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ReleaseRead(&cm_serverLock);
 }
 
 /*
@@ -301,36 +302,37 @@ cm_Analyze(cm_conn_t *connp,
            struct cm_fid *fidp,
            cm_cell_t *cellp,
            afs_uint32 storeOp,
+           AFSFetchStatus *statusp,
            AFSVolSync *volSyncp,
-           cm_serverRef_t * serversp,
+           cm_serverRef_t ** vlServerspp,
            cm_callbackRequest_t *cbrp,
            long errorCode)
 {
     cm_server_t *serverp = NULL;
-    cm_serverRef_t **serverspp = NULL;
+    cm_serverRef_t **volServerspp = NULL;
     cm_serverRef_t *tsrp;
     cm_ucell_t *ucellp;
     cm_volume_t * volp = NULL;
     cm_vol_state_t *statep = NULL;
     cm_scache_t * scp = NULL;
-    afs_uint32 replicated;
+    afs_uint32 replicated = 0;
     int retry = 0;
     int free_svr_list = 0;
-    int dead_session;
+    int dead_session = (userp->cellInfop == NULL);
     long timeUsed, timeLeft;
-    long code;
+    long code = 0;
     char addr[16]="unknown";
     int forcing_new = 0;
     int location_updated = 0;
     char *format;
     DWORD msgID;
+    int invalid_status = 0;
 
     osi_Log2(afsd_logp, "cm_Analyze connp 0x%p, code 0x%x",
              connp, errorCode);
 
     /* no locking required, since connp->serverp never changes after
      * creation */
-    dead_session = (userp->cellInfop == NULL);
     if (connp)
         serverp = connp->serverp;
 
@@ -354,30 +356,50 @@ cm_Analyze(cm_conn_t *connp,
 
     /* timeleft - get it from reqp the same way as cm_ConnByMServers does */
     timeUsed = (GetTickCount() - reqp->startTime) / 1000;
-    if ( reqp->flags & CM_REQ_SOURCE_SMB )
+    if ( (reqp->flags & CM_REQ_SOURCE_SMB) ||
+         (serverp && serverp->type == CM_SERVER_VLDB))
         timeLeft = HardDeadtimeout - timeUsed;
     else
         timeLeft = 0x0FFFFFFF;
 
+    /*
+     * Similar to the UNIX cache manager, if the AFSFetchStatus info
+     * returned by the file server is invalid, consider the response
+     * as being equivalent to VBUSY so that another file server can
+     * be queried if there is one.  If there is no replica, then the
+     * request will fail.
+     */
+    if (errorCode == 0 && statusp && !cm_IsStatusValid(statusp)) {
+        invalid_status = 1;
+        errorCode = VBUSY;
+    }
+
     /* get a pointer to the cell */
     if (errorCode) {
         if (cellp == NULL && serverp)
             cellp = serverp->cellp;
-        if (cellp == NULL && serversp) {
+        if (cellp == NULL && vlServerspp) {
             struct cm_serverRef * refp;
-            for ( refp=serversp ; cellp == NULL && refp != NULL; refp=refp->next) {
+            lock_ObtainRead(&cm_serverLock);
+            for ( refp=*vlServerspp ; cellp == NULL && refp != NULL; refp=refp->next) {
                 if (refp->status == srv_deleted)
                     continue;
                 if ( refp->server )
                     cellp = refp->server->cellp;
             }
+            lock_ReleaseRead(&cm_serverLock);
         }
         if (cellp == NULL && fidp) {
             cellp = cm_FindCellByID(fidp->cell, 0);
         }
     }
 
-    if (errorCode == CM_ERROR_TIMEDOUT) {
+    if (errorCode == 0) {
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+    }
+    else if (errorCode == CM_ERROR_TIMEDOUT) {
+       osi_Log0(afsd_logp, "cm_Analyze passed CM_ERROR_TIMEDOUT");
         if ( timeLeft > 5 ) {
             thrd_Sleep(3000);
             cm_CheckServers(CM_FLAG_CHECKDOWNSERVERS, cellp);
@@ -385,6 +407,11 @@ cm_Analyze(cm_conn_t *connp,
         }
     }
 
+    else if (errorCode == CM_ERROR_RETRY) {
+       osi_Log0(afsd_logp, "cm_Analyze passed CM_ERROR_RETRY");
+        retry = 1;
+    }
+
     else if (errorCode == UAEWOULDBLOCK || errorCode == EWOULDBLOCK ||
               errorCode == UAEAGAIN || errorCode == EAGAIN) {
        osi_Log0(afsd_logp, "cm_Analyze passed EWOULDBLOCK or EAGAIN.");
@@ -392,6 +419,9 @@ cm_Analyze(cm_conn_t *connp,
             thrd_Sleep(1000);
             retry = 1;
         }
+
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
     }
 
     /* if there is nosuchvolume, then we have a situation in which a
@@ -420,15 +450,18 @@ cm_Analyze(cm_conn_t *connp,
                                      CM_GETVOL_FLAG_NO_LRU_UPDATE,
                                      &volp);
             if (code == 0) {
+                lock_ObtainWrite(&volp->rw);
                 if (cm_UpdateVolumeLocation(cellp, userp, reqp, volp) == 0) {
-                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &serverspp);
+                    lock_ReleaseWrite(&volp->rw);
+                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &volServerspp);
                     if (code == 0) {
-                        if (!cm_IsServerListEmpty(*serverspp))
+                        if (!cm_IsServerListEmpty(*volServerspp))
                             retry = 1;
-                        cm_FreeServerList(serverspp, 0);
+                        cm_FreeServerList(volServerspp, 0);
                     }
+                } else {
+                    lock_ReleaseWrite(&volp->rw);
                 }
-
                 lock_ObtainRead(&cm_volumeLock);
                 cm_PutVolume(volp);
                 lock_ReleaseRead(&cm_volumeLock);
@@ -470,25 +503,25 @@ cm_Analyze(cm_conn_t *connp,
                                       CM_GETVOL_FLAG_NO_LRU_UPDATE,
                                       &volp);
             if (code == 0) {
-                if (!serversp) {
-                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &serverspp);
-                    if (code == 0) {
-                        serversp = *serverspp;
+                if (volServerspp == NULL) {
+                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &volServerspp);
+                    if (code == 0)
                         free_svr_list = 1;
-                    }
                 }
-                cm_ResetServerBusyStatus(serversp);
+                cm_ResetServerBusyStatus(volServerspp);
                 if (free_svr_list) {
-                    cm_FreeServerList(serverspp, 0);
+                    cm_FreeServerList(volServerspp, 0);
                     free_svr_list = 0;
-                    serversp = NULL;
+                    volServerspp = NULL;
                 }
 
                 /*
                  * Do not perform a cm_CheckOfflineVolume() if cm_Analyze()
                  * was called by cm_CheckOfflineVolumeState().
                  */
-                if (!(reqp->flags & CM_REQ_OFFLINE_VOL_CHK) && timeLeft > 7) {
+               if (!(reqp->flags & (CM_REQ_OFFLINE_VOL_CHK|CM_REQ_NORETRY)) &&
+                   timeLeft > 7)
+               {
                     thrd_Sleep(5000);
 
                     /* cm_CheckOfflineVolume() resets the serverRef state */
@@ -521,23 +554,26 @@ cm_Analyze(cm_conn_t *connp,
                                      CM_GETVOL_FLAG_NO_LRU_UPDATE,
                                      &volp);
             if (code == 0) {
-                if (!serversp) {
-                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &serverspp);
-                    if (code == 0) {
-                        serversp = *serverspp;
+                if (volServerspp == NULL) {
+                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &volServerspp);
+                    if (code == 0)
                         free_svr_list = 1;
-                    }
                 }
-                cm_ResetServerBusyStatus(serversp);
+                cm_ResetServerBusyStatus(volServerspp);
                 if (free_svr_list) {
-                    cm_FreeServerList(serverspp, 0);
+                    cm_FreeServerList(volServerspp, 0);
                     free_svr_list = 0;
-                    serversp = NULL;
+                    volServerspp = NULL;
                 }
 
-                if (timeLeft > 7) {
-                    thrd_Sleep(5000);
-                    statep = cm_VolumeStateByID(volp, fidp->volume);
+               /*
+                * retry all replicas for 5 minutes waiting 15 seconds
+                * between attempts.
+                */
+               if (timeLeft > 20 && !(reqp->flags & CM_REQ_NORETRY) &&
+                   reqp->volbusyCount++ < 20)
+               {
+                   thrd_Sleep(15000);
                     retry = 1;
                 }
                 cm_UpdateVolumeStatus(volp, fidp->volume);
@@ -550,30 +586,30 @@ cm_Analyze(cm_conn_t *connp,
         } else {    /* VL Server query */
             osi_Log0(afsd_logp, "cm_Analyze passed CM_ERROR_ALLBUSY (VL Server).");
 
-            if (timeLeft > 7) {
+           if (timeLeft > 7 && !(reqp->flags & CM_REQ_NORETRY) && vlServerspp)
+           {
                 thrd_Sleep(5000);
 
-                if (serversp) {
-                    cm_ResetServerBusyStatus(serversp);
-                    retry = 1;
-                }
+               cm_ResetServerBusyStatus(vlServerspp);
+               retry = 1;
             }
         }
     }
 
     /* special codes:  VBUSY and VRESTARTING */
     else if (errorCode == VBUSY || errorCode == VRESTARTING) {
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (fidp) {
             code = cm_FindVolumeByID(cellp, fidp->volume, userp, reqp,
                                       CM_GETVOL_FLAG_NO_LRU_UPDATE,
                                       &volp);
             if (code == 0) {
-                if (!serversp) {
-                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &serverspp);
-                    if (code == 0) {
-                        serversp = *serverspp;
+                if (volServerspp == NULL) {
+                    code = cm_GetVolServerList(volp, fidp->volume, userp, reqp, &replicated, &volServerspp);
+                    if (code == 0)
                         free_svr_list = 1;
-                    }
                 }
 
                 statep = cm_VolumeStateByID(volp, fidp->volume);
@@ -598,8 +634,13 @@ cm_Analyze(cm_conn_t *connp,
 
             switch ( errorCode ) {
             case VBUSY:
-                msgID = MSG_SERVER_REPORTS_VBUSY;
-                format = "Server %s reported busy when accessing volume %d in cell %s.";
+                if (invalid_status) {
+                    msgID = MSG_SERVER_REPLIED_BAD_STATUS;
+                    format = "Server %s replied with bad status info when accessing volume %d in cell %s.  Data discarded by cache manager.";
+                } else {
+                    msgID = MSG_SERVER_REPORTS_VBUSY;
+                    format = "Server %s reported busy when accessing volume %d in cell %s.";
+                }
                 break;
             case VRESTARTING:
                 msgID = MSG_SERVER_REPORTS_VRESTARTING;
@@ -610,12 +651,12 @@ cm_Analyze(cm_conn_t *connp,
             osi_Log3(afsd_logp, format, osi_LogSaveString(afsd_logp,addr), fidp->volume, cellp->name);
             LogEvent(EVENTLOG_WARNING_TYPE, msgID, addr, fidp->volume, cellp->name);
 
-            cm_SetServerBusyStatus(serversp, serverp);
+            cm_SetServerBusyStatus(volServerspp, serverp);
         }
 
         if (free_svr_list) {
-            cm_FreeServerList(serverspp, 0);
-            serversp = NULL;
+            cm_FreeServerList(volServerspp, 0);
+            volServerspp = NULL;
             free_svr_list = 0;
         }
         retry = 1;
@@ -625,6 +666,9 @@ cm_Analyze(cm_conn_t *connp,
     else if (errorCode == VNOVOL || errorCode == VMOVED || errorCode == VOFFLINE ||
              errorCode == VSALVAGE || errorCode == VIO)
     {
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         /* In case of timeout */
         reqp->volumeError = errorCode;
 
@@ -712,69 +756,75 @@ cm_Analyze(cm_conn_t *connp,
              * Mark server offline for this volume or delete the volume
              * from the server list if it was moved or is not present.
              */
-            if (!serversp || location_updated) {
-                code = cm_GetServerList(fidp, userp, reqp, &replicated, &serverspp);
-                if (code == 0) {
-                    serversp = *serverspp;
+            if (volServerspp == NULL || location_updated) {
+                code = cm_GetServerList(fidp, userp, reqp, &replicated, &volServerspp);
+                if (code == 0)
                     free_svr_list = 1;
-                }
             }
         }
 
-        lock_ObtainWrite(&cm_serverLock);
-        for (tsrp = serversp; tsrp; tsrp=tsrp->next) {
-            if (tsrp->status == srv_deleted)
-                continue;
 
-            sprintf(addr, "%d.%d.%d.%d",
-                     ((tsrp->server->addr.sin_addr.s_addr & 0xff)),
-                     ((tsrp->server->addr.sin_addr.s_addr & 0xff00)>> 8),
-                     ((tsrp->server->addr.sin_addr.s_addr & 0xff0000)>> 16),
-                     ((tsrp->server->addr.sin_addr.s_addr & 0xff000000)>> 24));
-
-            if (cm_ServerEqual(tsrp->server, serverp)) {
-                /* REDIRECT */
-                switch (errorCode) {
-                case VMOVED:
-                    osi_Log2(afsd_logp, "volume %u moved from server %s",
-                             fidp->volume, osi_LogSaveString(afsd_logp,addr));
-                    tsrp->status = srv_deleted;
-                    if (fidp)
-                        cm_RemoveVolumeFromServer(serverp, fidp->volume);
-                    break;
-                case VNOVOL:
-                    /*
-                     * The 1.6.0 and 1.6.1 file servers send transient VNOVOL errors which
-                     * are no indicative of the volume not being present.  For example,
-                     * VNOVOL can be sent during a transition to a VBUSY state prior to
-                     * salvaging or when cloning a .backup volume instance.  As a result
-                     * the cache manager must attempt at least one retry when a VNOVOL is
-                     * receive but there are no changes to the volume location information.
-                     */
-                    if (reqp->vnovolError > 0 && cm_ServerEqual(reqp->errorServp, serverp)) {
-                        osi_Log2(afsd_logp, "volume %u not present on server %s",
+        if (volServerspp) {
+            lock_ObtainWrite(&cm_serverLock);
+            for (tsrp = *volServerspp; tsrp; tsrp=tsrp->next) {
+                if (tsrp->status == srv_deleted)
+                    continue;
+
+                sprintf(addr, "%d.%d.%d.%d",
+                         ((tsrp->server->addr.sin_addr.s_addr & 0xff)),
+                         ((tsrp->server->addr.sin_addr.s_addr & 0xff00)>> 8),
+                         ((tsrp->server->addr.sin_addr.s_addr & 0xff0000)>> 16),
+                         ((tsrp->server->addr.sin_addr.s_addr & 0xff000000)>> 24));
+
+                if (cm_ServerEqual(tsrp->server, serverp)) {
+                    /* REDIRECT */
+                    switch (errorCode) {
+                    case VMOVED:
+                        osi_Log2(afsd_logp, "volume %u moved from server %s",
                                   fidp->volume, osi_LogSaveString(afsd_logp,addr));
                         tsrp->status = srv_deleted;
                         if (fidp)
                             cm_RemoveVolumeFromServer(serverp, fidp->volume);
-                    } else {
-                        osi_Log2(afsd_logp, "VNOVOL received for volume %u from server %s",
-                                 fidp->volume, osi_LogSaveString(afsd_logp,addr));
-                        if (replicated) {
-                            if (tsrp->status == srv_not_busy)
-                                tsrp->status = srv_busy;
+                        break;
+                    case VNOVOL:
+                        /*
+                        * The 1.6.0 and 1.6.1 file servers send transient VNOVOL errors which
+                        * are no indicative of the volume not being present.  For example,
+                        * VNOVOL can be sent during a transition to a VBUSY state prior to
+                        * salvaging or when cloning a .backup volume instance.  As a result
+                        * the cache manager must attempt at least one retry when a VNOVOL is
+                        * receive but there are no changes to the volume location information.
+                        */
+                        if (reqp->vnovolError > 0 && cm_ServerEqual(reqp->errorServp, serverp)) {
+                            osi_Log2(afsd_logp, "volume %u not present on server %s",
+                                      fidp->volume, osi_LogSaveString(afsd_logp,addr));
+                            tsrp->status = srv_deleted;
+                            if (fidp)
+                                cm_RemoveVolumeFromServer(serverp, fidp->volume);
                         } else {
-                            Sleep(2000);
+                            osi_Log2(afsd_logp, "VNOVOL received for volume %u from server %s",
+                                      fidp->volume, osi_LogSaveString(afsd_logp,addr));
+                            if (replicated) {
+                                if (tsrp->status == srv_not_busy)
+                                    tsrp->status = srv_busy;
+                            } else {
+                                Sleep(2000);
+                            }
                         }
+                        break;
+                    case VOFFLINE:
+                        osi_Log2(afsd_logp, "VOFFLINE received for volume %u from server %s",
+                                  fidp->volume, osi_LogSaveString(afsd_logp,addr));
+                        tsrp->status = srv_offline;
+                        break;
+                    default:
+                        osi_Log3(afsd_logp, "volume %u exists on server %s with status %u",
+                                  fidp->volume, osi_LogSaveString(afsd_logp,addr), tsrp->status);
                     }
-                    break;
-                default:
-                    osi_Log3(afsd_logp, "volume %u exists on server %s with status %u",
-                             fidp->volume, osi_LogSaveString(afsd_logp,addr), tsrp->status);
                 }
             }
+            lock_ReleaseWrite(&cm_serverLock);
         }
-        lock_ReleaseWrite(&cm_serverLock);
 
         /* Remember that the VNOVOL error occurred */
         if (errorCode == VNOVOL) {
@@ -782,16 +832,25 @@ cm_Analyze(cm_conn_t *connp,
             reqp->vnovolError++;
         }
 
+        /* Remember that the VIO error occurred */
+        if (errorCode == VIO) {
+            reqp->errorServp = serverp;
+            reqp->vioCount++;
+        }
+
         /* Free the server list before cm_ForceUpdateVolume is called */
         if (free_svr_list) {
-            cm_FreeServerList(serverspp, 0);
-            serversp = NULL;
+            cm_FreeServerList(volServerspp, 0);
+            volServerspp = NULL;
             free_svr_list = 0;
         }
 
-        if ( timeLeft > 2 )
+        if ( timeLeft > 2 && reqp->vioCount < 100)
             retry = 1;
     } else if ( errorCode == VNOVNODE ) {
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
        if ( fidp ) {
            osi_Log4(afsd_logp, "cm_Analyze passed VNOVNODE cell %u vol %u vn %u uniq %u.",
                      fidp->cell, fidp->volume, fidp->vnode, fidp->unique);
@@ -916,6 +975,9 @@ cm_Analyze(cm_conn_t *connp,
          * is currently busy on the server.  Unconditionally
          * retry the request so an alternate call channel can be used.
          */
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (serverp)
             sprintf(addr, "%d.%d.%d.%d",
                     ((serverp->addr.sin_addr.s_addr & 0xff)),
@@ -939,6 +1001,9 @@ cm_Analyze(cm_conn_t *connp,
          * The RPC was not serviced so it can be retried and any
          * existing status information is still valid.
          */
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (fidp) {
             if (serverp)
                 sprintf(addr, "%d.%d.%d.%d",
@@ -979,6 +1044,9 @@ cm_Analyze(cm_conn_t *connp,
          * client should fail over to another server.  If this is a
          * request against a single source, the client may retry once.
          */
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (serverp)
             sprintf(addr, "%d.%d.%d.%d",
                     ((serverp->addr.sin_addr.s_addr & 0xff)),
@@ -1047,7 +1115,8 @@ cm_Analyze(cm_conn_t *connp,
                  (reqp->flags & CM_REQ_NEW_CONN_FORCED ? "yes" : "no"));
 
         if (serverp) {
-            if ((reqp->flags & CM_REQ_NEW_CONN_FORCED)) {
+           if ((connp->flags & CM_CONN_FLAG_NEW) ||
+               (reqp->flags & CM_REQ_NEW_CONN_FORCED)) {
                 lock_ObtainMutex(&serverp->mx);
                 if (!(serverp->flags & CM_SERVERFLAG_DOWN)) {
                     _InterlockedOr(&serverp->flags, CM_SERVERFLAG_DOWN);
@@ -1097,7 +1166,8 @@ cm_Analyze(cm_conn_t *connp,
                  (reqp->flags & CM_REQ_NEW_CONN_FORCED ? "yes" : "no"));
 
         if (serverp) {
-            if (reqp->flags & CM_REQ_NEW_CONN_FORCED) {
+           if ((connp->flags & CM_CONN_FLAG_NEW) ||
+               (reqp->flags & CM_REQ_NEW_CONN_FORCED)) {
                 reqp->errorServp = serverp;
                 reqp->tokenError = errorCode;
             } else {
@@ -1105,9 +1175,10 @@ cm_Analyze(cm_conn_t *connp,
                 forcing_new = 1;
                 cm_ForceNewConnections(serverp);
             }
+
+            if ( timeLeft > 2 )
+                retry = 1;
         }
-        if ( timeLeft > 2 )
-            retry = 1;
     }
     else if (errorCode == RXKADEXPIRED) {
         osi_Log1(afsd_logp, "cm_Analyze: rxkad error code 0x%x (RXKADEXPIRED)",
@@ -1122,6 +1193,11 @@ cm_Analyze(cm_conn_t *connp,
             _InterlockedAnd(&ucellp->flags, ~CM_UCELLFLAG_RXKAD);
             ucellp->gen++;
             lock_ReleaseMutex(&userp->mx);
+
+            reqp->flags |= CM_REQ_NEW_CONN_FORCED;
+            forcing_new = 1;
+            cm_ForceNewConnections(serverp);
+
             if ( timeLeft > 2 )
                 retry = 1;
         }
@@ -1145,6 +1221,9 @@ cm_Analyze(cm_conn_t *connp,
         osi_Log2(afsd_logp, "cm_Analyze: rxkad error code 0x%x (%s)",
                   errorCode, s);
 
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (serverp) {
             reqp->errorServp = serverp;
             reqp->tokenError = errorCode;
@@ -1158,16 +1237,24 @@ cm_Analyze(cm_conn_t *connp,
          * to answer our query.  Therefore, we will retry the request
          * and force the use of another server.
          */
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (serverp) {
             reqp->errorServp = serverp;
             reqp->tokenError = errorCode;
             retry = 1;
         }
     } else if (errorCode == VICECONNBAD || errorCode == VICETOKENDEAD) {
-       cm_ForceNewConnections(serverp);
+        reqp->flags |= CM_REQ_NEW_CONN_FORCED;
+        forcing_new = 1;
+        cm_ForceNewConnections(serverp);
         if ( timeLeft > 2 )
             retry = 1;
     } else {
+       if (connp)
+           _InterlockedAnd(&connp->flags, ~CM_CONN_FLAG_NEW);
+
         if (errorCode) {
             char * s = "unknown error";
             switch ( errorCode ) {
@@ -1234,8 +1321,6 @@ cm_Analyze(cm_conn_t *connp,
             case VL_BADMASK        : s = "VL_BADMASK";         break;
            case CM_ERROR_NOSUCHCELL        : s = "CM_ERROR_NOSUCHCELL";         break;
            case CM_ERROR_NOSUCHVOLUME      : s = "CM_ERROR_NOSUCHVOLUME";       break;
-           case CM_ERROR_TIMEDOUT          : s = "CM_ERROR_TIMEDOUT";           break;
-           case CM_ERROR_RETRY             : s = "CM_ERROR_RETRY";              break;
            case CM_ERROR_NOACCESS          : s = "CM_ERROR_NOACCESS";           break;
            case CM_ERROR_NOSUCHFILE        : s = "CM_ERROR_NOSUCHFILE";         break;
            case CM_ERROR_STOPNOW           : s = "CM_ERROR_STOPNOW";            break;
@@ -1283,6 +1368,8 @@ cm_Analyze(cm_conn_t *connp,
            case CM_ERROR_ALLDOWN           : s = "CM_ERROR_ALLDOWN";            break;
            case CM_ERROR_TOOFEWBUFS        : s = "CM_ERROR_TOOFEWBUFS";         break;
            case CM_ERROR_TOOMANYBUFS       : s = "CM_ERROR_TOOMANYBUFS";        break;
+            case UAEIO                      : s = "UAEIO";                       break;
+            case EIO                        : s = "EIO";                         break;
             }
             osi_Log2(afsd_logp, "cm_Analyze: ignoring error code 0x%x (%s)",
                      errorCode, s);
@@ -1291,10 +1378,9 @@ cm_Analyze(cm_conn_t *connp,
     }
 
     /* If not allowed to retry, don't */
-    if (!forcing_new && (reqp->flags & CM_REQ_NORETRY) &&
-        (errorCode != RX_MSGSIZE && errorCode != RX_CALL_BUSY))
-       retry = 0;
-    else if (retry && dead_session)
+    if (dead_session ||
+        !forcing_new && (reqp->flags & CM_REQ_NORETRY) &&
+        !(errorCode > -64 && errorCode <= RX_INVALID_OPERATION))
         retry = 0;
 
     /* drop this on the way out */
@@ -1302,6 +1388,7 @@ cm_Analyze(cm_conn_t *connp,
         cm_PutConn(connp);
 
     /*
+
      * clear the volume updated flag if we succeed.
      * this way the flag will not prevent a subsequent volume
      * from being updated if necessary.
@@ -1311,12 +1398,12 @@ cm_Analyze(cm_conn_t *connp,
         reqp->flags &= ~CM_REQ_VOLUME_UPDATED;
     }
 
-    if ( serversp &&
+    if ( vlServerspp &&
          errorCode != VBUSY &&
          errorCode != VRESTARTING &&
          errorCode != CM_ERROR_ALLBUSY)
     {
-        cm_ResetServerBusyStatus(serversp);
+        cm_ResetServerBusyStatus(vlServerspp);
     }
 
     /* retry until we fail to find a connection */
@@ -1520,13 +1607,14 @@ static void cm_NewRXConnection(cm_conn_t *tcp, cm_ucell_t *ucellp,
                                     secObjp,
                                     secIndex);
     rx_SetConnDeadTime(tcp->rxconnp, ConnDeadtimeout);
-    rx_SetConnHardDeadTime(tcp->rxconnp, HardDeadtimeout);
+    if (smb_Enabled || tcp->serverp->type == CM_SERVER_VLDB)
+        rx_SetConnHardDeadTime(tcp->rxconnp, HardDeadtimeout);
 
     /*
      * Setting idle dead timeout to a non-zero value activates RX_CALL_IDLE errors
      */
     if (replicated) {
-        tcp->flags &= CM_CONN_FLAG_REPLICATION;
+       _InterlockedOr(&tcp->flags, CM_CONN_FLAG_REPLICATION);
         rx_SetConnIdleDeadTime(tcp->rxconnp, ReplicaIdleDeadtimeout);
     } else {
         rx_SetConnIdleDeadTime(tcp->rxconnp, IdleDeadtimeout);
@@ -1552,6 +1640,9 @@ static void cm_NewRXConnection(cm_conn_t *tcp, cm_ucell_t *ucellp,
     tcp->ucgen = ucellp->gen;
     if (secObjp)
         rxs_Release(secObjp);   /* Decrement the initial refCount */
+
+    _InterlockedAnd(&tcp->flags, ~CM_CONN_FLAG_FORCE_NEW);
+    _InterlockedOr(&tcp->flags, CM_CONN_FLAG_NEW);
 }
 
 long cm_ConnByServer(cm_server_t *serverp, cm_user_t *userp, afs_uint32 replicated, cm_conn_t **connpp)
@@ -1582,6 +1673,7 @@ long cm_ConnByServer(cm_server_t *serverp, cm_user_t *userp, afs_uint32 replicat
                 break;
         }
         if (tcp) {
+            InterlockedIncrement(&tcp->refCount);
             lock_ReleaseWrite(&cm_connLock);
             goto haveconn;
         }
@@ -1600,10 +1692,10 @@ long cm_ConnByServer(cm_server_t *serverp, cm_user_t *userp, afs_uint32 replicat
         lock_ReleaseWrite(&cm_connLock);
         lock_ReleaseMutex(&userp->mx);
     } else {
+        InterlockedIncrement(&tcp->refCount);
         lock_ReleaseRead(&cm_connLock);
       haveconn:
         lock_ReleaseMutex(&userp->mx);
-        InterlockedIncrement(&tcp->refCount);
 
         lock_ObtainMutex(&tcp->mx);
         if ((tcp->flags & CM_CONN_FLAG_FORCE_NEW) ||
@@ -1614,7 +1706,6 @@ long cm_ConnByServer(cm_server_t *serverp, cm_user_t *userp, afs_uint32 replicat
                 osi_Log0(afsd_logp, "cm_ConnByServer replace connection due to token update");
             else
                 osi_Log0(afsd_logp, "cm_ConnByServer replace connection due to crypt change");
-            tcp->flags &= ~CM_CONN_FLAG_FORCE_NEW;
             rx_SetConnSecondsUntilNatPing(tcp->rxconnp, 0);
             rx_DestroyConnection(tcp->rxconnp);
             cm_NewRXConnection(tcp, ucellp, serverp, replicated);
@@ -1737,7 +1828,7 @@ void cm_ForceNewConnections(cm_server_t *serverp)
     lock_ObtainWrite(&cm_connLock);
     for (tcp = serverp->connsp; tcp; tcp=tcp->nextp) {
        lock_ObtainMutex(&tcp->mx);
-       tcp->flags |= CM_CONN_FLAG_FORCE_NEW;
+       _InterlockedOr(&tcp->flags, CM_CONN_FLAG_FORCE_NEW);
        lock_ReleaseMutex(&tcp->mx);
     }
     lock_ReleaseWrite(&cm_connLock);