Windows: Record callback issued time
[openafs.git] / src / WINNT / afsd / cm_callback.c
index e919bf4..1936698 100644 (file)
@@ -1,13 +1,16 @@
 /*
  * Copyright 2000, International Business Machines Corporation and others.
  * All Rights Reserved.
- * 
+ *
  * This software has been released under the terms of the IBM Public
  * License.  For details, see the LICENSE file in the top-level source
  * directory or online at http://www.openafs.org/dl/license10.html
  */
 
+#include <afsconfig.h>
 #include <afs/param.h>
+#include <roken.h>
+
 #include <afs/afs_args.h>
 #include <afs/stds.h>
 
@@ -33,7 +36,7 @@ osi_rwlock_t cm_callbackLock;
 
 afs_int32 cm_OfflineROIsValid = 0;
 
-afs_int32 cm_giveUpAllCBs = 0;
+afs_int32 cm_giveUpAllCBs = 1;
 
 afs_int32 cm_shutdown = 0;
 
@@ -106,7 +109,7 @@ void cm_CallbackNotifyChange(cm_scache_t *scp)
     DWORD dummyLen;
 
     /* why does this have to query the registry each time? */
-       if (RegOpenKeyEx( HKEY_LOCAL_MACHINE, 
+       if (RegOpenKeyEx( HKEY_LOCAL_MACHINE,
                       AFSREG_CLT_OPENAFS_SUBKEY,
                       0,
                       KEY_READ|KEY_QUERY_VALUE,
@@ -119,7 +122,7 @@ void cm_CallbackNotifyChange(cm_scache_t *scp)
     }
 
     if (dwDelay > 5000)    /* do not allow a delay of more then 5 seconds */
-        dwDelay = 5000;   
+        dwDelay = 5000;
 
     osi_Log3(afsd_logp, "CallbackNotifyChange FileType %d Flags %lX Delay %dms",
               scp->fileType, scp->flags, dwDelay);
@@ -145,7 +148,7 @@ void cm_CallbackNotifyChange(cm_scache_t *scp)
             smb_NotifyChange( 0,
                               FILE_NOTIFY_GENERIC_FILE_FILTER,
                               dscp, NULL, NULL, TRUE);
-        if (dscp) 
+        if (dscp)
             cm_ReleaseSCache(dscp);
     }
 }
@@ -161,16 +164,17 @@ void cm_RevokeCallback(struct rx_call *callp, cm_cell_t * cellp, AFSFid *fidp)
     cm_fid_t tfid;
     cm_scache_t *scp;
     long hash;
-        
+
     tfid.cell = cellp ? cellp->cellID : 0;
     tfid.volume = fidp->Volume;
     tfid.vnode = fidp->Vnode;
     tfid.unique = fidp->Unique;
+    CM_FID_GEN_HASH(&tfid);
     hash = CM_SCACHE_HASH(&tfid);
 
     osi_Log3(afsd_logp, "RevokeCallback vol %u vn %u uniq %u",
              fidp->Volume, fidp->Vnode, fidp->Unique);
-        
+
     /* do this first, so that if we're executing a callback granting call
      * at this moment, we kill it before it can be merged in.  Otherwise,
      * it could complete while we're doing the scan below, and get missed
@@ -191,15 +195,19 @@ void cm_RevokeCallback(struct rx_call *callp, cm_cell_t * cellp, AFSFid *fidp)
         {
             cm_HoldSCacheNoLock(scp);
             lock_ReleaseWrite(&cm_scacheLock);
-            osi_Log4(afsd_logp, "RevokeCallback Discarding SCache scp 0x%p vol %u vn %u uniq %u", 
+            osi_Log4(afsd_logp, "RevokeCallback Discarding SCache scp 0x%p vol %u vn %u uniq %u",
                      scp, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
 
             lock_ObtainWrite(&scp->rw);
             cm_DiscardSCache(scp);
             lock_ReleaseWrite(&scp->rw);
 
+            if (RDR_Initialized)
+                RDR_InvalidateObject(scp->fid.cell, scp->fid.volume, scp->fid.vnode, scp->fid.unique,
+                                     scp->fid.hash, scp->fileType, AFS_INVALIDATE_CALLBACK);
+
             cm_CallbackNotifyChange(scp);
-            
+
             lock_ObtainWrite(&cm_scacheLock);
             cm_ReleaseSCacheNoLock(scp);
         }
@@ -210,6 +218,24 @@ void cm_RevokeCallback(struct rx_call *callp, cm_cell_t * cellp, AFSFid *fidp)
              fidp->Volume, fidp->Vnode, fidp->Unique);
 }
 
+static __inline void
+cm_callbackDiscardROVolumeByFID(cm_fid_t *fidp)
+{
+    cm_volume_t *volp = cm_GetVolumeByFID(fidp);
+    if (volp) {
+        cm_PutVolume(volp);
+        if (volp->cbExpiresRO) {
+            volp->cbExpiresRO = 0;
+            volp->cbIssuedRO = 0;
+            if (volp->cbServerpRO) {
+                cm_PutServer(volp->cbServerpRO);
+                volp->cbServerpRO = NULL;
+            }
+            volp->creationDateRO = 0;
+        }
+    }
+}
+
 /* called to revoke a volume callback, which is typically issued when a volume
  * is moved from one server to another.
  *
@@ -221,7 +247,7 @@ void cm_RevokeVolumeCallback(struct rx_call *callp, cm_cell_t *cellp, AFSFid *fi
     cm_scache_t *scp;
     cm_fid_t tfid;
 
-    osi_Log1(afsd_logp, "RevokeVolumeCallback vol %d", fidp->Volume);
+    osi_Log1(afsd_logp, "RevokeVolumeCallback vol %u", fidp->Volume);
 
     /* do this first, so that if we're executing a callback granting call
      * at this moment, we kill it before it can be merged in.  Otherwise,
@@ -231,7 +257,6 @@ void cm_RevokeVolumeCallback(struct rx_call *callp, cm_cell_t *cellp, AFSFid *fi
     tfid.cell = cellp ? cellp->cellID : 0;
     tfid.volume = fidp->Volume;
     tfid.vnode = tfid.unique = 0;
-
     cm_RecordRacingRevoke(&tfid, CM_RACINGFLAG_CANCELVOL);
 
     lock_ObtainWrite(&cm_scacheLock);
@@ -245,27 +270,33 @@ void cm_RevokeVolumeCallback(struct rx_call *callp, cm_cell_t *cellp, AFSFid *fi
                 lock_ReleaseWrite(&cm_scacheLock);
 
                 lock_ObtainWrite(&scp->rw);
-                osi_Log4(afsd_logp, "RevokeVolumeCallback Discarding SCache scp 0x%p vol %u vn %u uniq %u", 
-                          scp, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
+                osi_Log5(afsd_logp, "RevokeVolumeCallback Discarding SCache scp 0x%p vol %u vn %u uniq %u",
+                          scp, scp->fid.cell, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
+                osi_Log2(afsd_logp, ".... dv 0x%x:%x",
+                          (afs_uint32)((scp->dataVersion >> 32) & 0xFFFFFFFF),
+                          (afs_uint32)(scp->dataVersion & 0xFFFFFFFF));
+
                 cm_DiscardSCache(scp);
                 lock_ReleaseWrite(&scp->rw);
 
+                if (RDR_Initialized)
+                    RDR_InvalidateObject(scp->fid.cell, scp->fid.volume, scp->fid.vnode, scp->fid.unique,
+                                         scp->fid.hash, scp->fileType, AFS_INVALIDATE_CALLBACK);
+
                 cm_CallbackNotifyChange(scp);
                 lock_ObtainWrite(&cm_scacheLock);
                 cm_ReleaseSCacheNoLock(scp);
-                if (scp->flags & CM_SCACHEFLAG_PURERO) {
-                    cm_volume_t *volp = cm_GetVolumeByFID(&scp->fid);
-                    if (volp) {
-                        volp->cbExpiresRO = 0;
-                        cm_PutVolume(volp);
-                    }
-                }
+                if (scp->flags & CM_SCACHEFLAG_PURERO)
+                    cm_callbackDiscardROVolumeByFID(&scp->fid);
             }
         }      /* search one hash bucket */
     }  /* search all hash buckets */
 
     lock_ReleaseWrite(&cm_scacheLock);
 
+    if (cellp && RDR_Initialized)
+        RDR_InvalidateVolume(cellp->cellID, fidp->Volume, AFS_INVALIDATE_CALLBACK);
+
     osi_Log1(afsd_logp, "RevokeVolumeCallback Complete vol %d", fidp->Volume);
 }
 
@@ -354,7 +385,7 @@ SRXAFSCB_CallBack(struct rx_call *callp, AFSCBFids *fidsArrayp, AFSCBs *cbsArray
         host = rx_HostOf(peerp);
         port = rx_PortOf(peerp);
 
-        tsp = cm_FindServerByIP(host, port, CM_SERVER_FILE);
+        tsp = cm_FindServerByIP(host, port, CM_SERVER_FILE, FALSE);
         if (tsp) {
             cellp = tsp->cellp;
             cm_PutServer(tsp);
@@ -364,7 +395,7 @@ SRXAFSCB_CallBack(struct rx_call *callp, AFSCBFids *fidsArrayp, AFSCBs *cbsArray
             osi_Log2(afsd_logp, "SRXAFSCB_CallBack from host 0x%x port %d",
                      ntohl(host),
                      ntohs(port));
-        else 
+        else
             osi_Log3(afsd_logp, "SRXAFSCB_CallBack from host 0x%x port %d for cell %s",
                      ntohl(host),
                      ntohs(port),
@@ -376,7 +407,7 @@ SRXAFSCB_CallBack(struct rx_call *callp, AFSCBFids *fidsArrayp, AFSCBs *cbsArray
 
     for (i=0; i < (long) fidsArrayp->AFSCBFids_len; i++) {
         tfidp = &fidsArrayp->AFSCBFids_val[i];
-                
+
         if (tfidp->Volume == 0)
             continue;   /* means don't do anything */
         else if (tfidp->Vnode == 0)
@@ -493,7 +524,6 @@ extern osi_rwlock_t cm_aclLock;
 extern osi_rwlock_t buf_globalLock;
 extern osi_rwlock_t cm_cellLock;
 extern osi_rwlock_t cm_connLock;
-extern osi_rwlock_t cm_daemonLock;
 extern osi_rwlock_t cm_dnlcLock;
 extern osi_rwlock_t cm_scacheLock;
 extern osi_rwlock_t cm_serverLock;
@@ -528,7 +558,6 @@ static struct _ltable {
     {"cm_connLock",      (char*)&cm_connLock,           LOCKTYPE_RW},
     {"cm_userLock",      (char*)&cm_userLock,           LOCKTYPE_RW},
     {"cm_volumeLock",    (char*)&cm_volumeLock,         LOCKTYPE_RW},
-    {"cm_daemonLock",    (char*)&cm_daemonLock,         LOCKTYPE_RW},
     {"cm_dnlcLock",      (char*)&cm_dnlcLock,           LOCKTYPE_RW},
     {"cm_utilsLock",     (char*)&cm_utilsLock,          LOCKTYPE_RW},
     {"smb_globalLock",   (char*)&smb_globalLock,        LOCKTYPE_RW},
@@ -562,7 +591,7 @@ SRXAFSCB_GetLock(struct rx_call *callp, long index, AFSDBLock *lockp)
         port = rx_PortOf(peerp);
     }
 
-    osi_Log3(afsd_logp, "SRXAFSCB_GetLock(%d) from host 0x%x port %d", 
+    osi_Log3(afsd_logp, "SRXAFSCB_GetLock(%d) from host 0x%x port %d",
              index, ntohl(host), ntohs(port));
 
     nentries = sizeof(ltable) / sizeof(struct _ltable);
@@ -772,12 +801,7 @@ SRXAFSCB_GetCE64(struct rx_call *callp, long index, AFSDBCacheEntry64 *cep)
     cep->lock.pid_last_reader = 0;
     cep->lock.pid_writer = 0;
     cep->lock.src_indicator = 0;
-#if !defined(AFS_64BIT_ENV)
-    cep->Length.high = scp->length.HighPart;
-    cep->Length.low = scp->length.LowPart;
-#else
     cep->Length = (afs_int64) scp->length.QuadPart;
-#endif
     cep->DataVersion = (afs_uint32)(scp->dataVersion & 0xFFFFFFFF);
     cep->callback = afs_data_pointer_to_int32(scp->cbServerp);
     if (scp->flags & CM_SCACHEFLAG_PURERO) {
@@ -939,7 +963,7 @@ SRXAFSCB_WhoAreYou(struct rx_call *callp, struct interfaceAddr* addr)
     for ( i=0; i < cm_noIPAddr; i++ ) {
         addr->addr_in[i] = cm_IPAddr[i];
         addr->subnetmask[i] = cm_SubnetMask[i];
-        addr->mtu[i] = (rx_mtu == -1 || (rx_mtu != -1 && cm_NetMtu[i] < rx_mtu)) ? 
+        addr->mtu[i] = (rx_mtu == -1 || (rx_mtu != -1 && cm_NetMtu[i] < rx_mtu)) ?
             cm_NetMtu[i] : rx_mtu;
     }
 
@@ -953,12 +977,13 @@ SRXAFSCB_InitCallBackState3(struct rx_call *callp, afsUUID* serverUuid)
 {
     char *p = NULL;
 
-    struct sockaddr_in taddr;
     cm_server_t *tsp = NULL;
     cm_scache_t *scp = NULL;
     cm_cell_t* cellp = NULL;
     afs_uint32 hash;
     int discarded;
+    cm_fid_t discardFid;
+    afs_uint32 discardType;
     struct rx_connection *connp;
     struct rx_peer *peerp;
     unsigned long host = 0;
@@ -975,12 +1000,12 @@ SRXAFSCB_InitCallBackState3(struct rx_call *callp, afsUUID* serverUuid)
             if (UuidToString((UUID *)serverUuid, &p) == RPC_S_OK) {
                 osi_Log1(afsd_logp, "SRXAFSCB_InitCallBackState3 Uuid%s ->",osi_LogSaveString(afsd_logp,p));
                 RpcStringFree(&p);
-            } 
+            }
 
-            tsp = cm_FindServerByUuid(serverUuid, CM_SERVER_FILE);
+            tsp = cm_FindServerByUuid(serverUuid, CM_SERVER_FILE, FALSE);
         }
         if (!tsp)
-            tsp = cm_FindServerByIP(host, port, CM_SERVER_FILE);
+            tsp = cm_FindServerByIP(host, port, CM_SERVER_FILE, FALSE);
         if (tsp) {
             cellp = tsp->cellp;
             cm_PutServer(tsp);
@@ -990,7 +1015,7 @@ SRXAFSCB_InitCallBackState3(struct rx_call *callp, afsUUID* serverUuid)
             osi_Log2(afsd_logp, "SRXAFSCB_InitCallBackState3 from host 0x%x port %d",
                      ntohl(host),
                      ntohs(port));
-        else 
+        else
             osi_Log3(afsd_logp, "SRXAFSCB_InitCallBackState3 from host 0x%x port %d for cell %s",
                      ntohl(host),
                      ntohs(port),
@@ -1000,14 +1025,10 @@ SRXAFSCB_InitCallBackState3(struct rx_call *callp, afsUUID* serverUuid)
     }
 
     if (connp && peerp) {
-       taddr.sin_family = AF_INET;
-       taddr.sin_addr.s_addr = rx_HostOf(rx_PeerOf(rx_ConnectionOf(callp)));
-       taddr.sin_port = rx_PortOf(rx_PeerOf(rx_ConnectionOf(callp)));
-
-       tsp = cm_FindServer(&taddr, CM_SERVER_FILE);
+       tsp = cm_FindServerByIP(host, port, CM_SERVER_FILE, FALSE);
 
        osi_Log1(afsd_logp, "InitCallbackState3 server %x", tsp);
-       
+
        /* record the callback in the racing revokes structure.  This
         * shouldn't be necessary, since we shouldn't be making callback
         * granting calls while we're going to get an initstate call,
@@ -1051,32 +1072,32 @@ SRXAFSCB_InitCallBackState3(struct rx_call *callp, afsUUID* serverUuid)
                 if (scp->cbExpires > 0 && scp->cbServerp != NULL) {
                     /* we have a callback, now decide if we should clear it */
                     if (cm_ServerEqual(scp->cbServerp, tsp)) {
-                        osi_Log4(afsd_logp, "InitCallbackState3 Discarding SCache scp 0x%p vol %u vn %u uniq %u", 
+                        osi_Log4(afsd_logp, "InitCallbackState3 Discarding SCache scp 0x%p vol %u vn %u uniq %u",
                                   scp, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
+                        discardFid = scp->fid;
+                        discardType = scp->fileType;
                         cm_DiscardSCache(scp);
                         discarded = 1;
                     }
                 }
                 lock_ReleaseWrite(&scp->rw);
-                if (discarded)
+                if (discarded) {
                     cm_CallbackNotifyChange(scp);
+                    if (RDR_Initialized)
+                        RDR_InvalidateObject(discardFid.cell, discardFid.volume, discardFid.vnode, discardFid.unique,
+                                             discardFid.hash, discardType, AFS_INVALIDATE_EXPIRED);
+                }
                 lock_ObtainWrite(&cm_scacheLock);
                 cm_ReleaseSCacheNoLock(scp);
 
-                if (discarded && (scp->flags & CM_SCACHEFLAG_PURERO)) {
-                    cm_volume_t *volp = cm_GetVolumeByFID(&scp->fid);
-                    if (volp) {
-                        if (volp->cbExpiresRO != 0)
-                            volp->cbExpiresRO = 0;
-                        cm_PutVolume(volp);
-                    }
-                }
+                if (discarded && (scp->flags & CM_SCACHEFLAG_PURERO))
+                    cm_callbackDiscardROVolumeByFID(&scp->fid);
 
             }  /* search one hash bucket */
        }       /* search all hash buckets */
-       
+
        lock_ReleaseWrite(&cm_scacheLock);
-       
+
        if (tsp) {
            /* reset the No flags on the server */
            cm_SetServerNo64Bit(tsp, 0);
@@ -1084,7 +1105,7 @@ SRXAFSCB_InitCallBackState3(struct rx_call *callp, afsUUID* serverUuid)
 
            /* we're done with the server structure */
             cm_PutServer(tsp);
-       } 
+       }
     }
     return 0;
 }
@@ -1111,8 +1132,8 @@ SRXAFSCB_ProbeUuid(struct rx_call *callp, afsUUID* clientUuid)
     if ( !afs_uuid_equal(&cm_data.Uuid, clientUuid) ) {
         UuidToString((UUID *)&cm_data.Uuid, &p);
         UuidToString((UUID *)clientUuid, &q);
-        osi_Log4(afsd_logp, "SRXAFSCB_ProbeUuid %s != %s from host 0x%x port %d", 
-                  osi_LogSaveString(afsd_logp,p), 
+        osi_Log4(afsd_logp, "SRXAFSCB_ProbeUuid %s != %s from host 0x%x port %d",
+                  osi_LogSaveString(afsd_logp,p),
                   osi_LogSaveString(afsd_logp,q),
                   ntohl(host),
                   ntohs(port));
@@ -1129,30 +1150,36 @@ SRXAFSCB_ProbeUuid(struct rx_call *callp, afsUUID* clientUuid)
 }
 
 /* debug interface */
-static int 
+static int
 GetCellCommon(afs_int32 a_cellnum, char **a_name, serverList *a_hosts)
 {
     afs_int32 sn;
     cm_cell_t * cellp;
-    cm_serverRef_t * serverRefp; 
+    cm_serverRef_t * serverRefp;
+    size_t len;
 
     cellp = cm_FindCellByID(a_cellnum, CM_FLAG_NOPROBE);
     if (!cellp) {
-        *a_name = strdup("");
+        *a_name = (char *)xdr_alloc(sizeof(char));
+        if (*a_name)
+            *a_name = '\0';
         return 0;
     }
 
     lock_ObtainRead(&cm_serverLock);
-    *a_name = strdup(cellp->name);
+    len = strlen(cellp->name)+1;
+    *a_name = (char *)xdr_alloc(len);
+    if (*a_name)
+        memcpy(*a_name, cellp->name, len);
 
-    for ( sn = 0, serverRefp = cellp->vlServersp; 
+    for ( sn = 0, serverRefp = cellp->vlServersp;
           sn < AFSMAXCELLHOSTS && serverRefp;
           sn++, serverRefp = serverRefp->next);
 
     a_hosts->serverList_len = sn;
     a_hosts->serverList_val = (afs_int32 *)xdr_alloc(sn * sizeof(afs_int32));
 
-    for ( sn = 0, serverRefp = cellp->vlServersp; 
+    for ( sn = 0, serverRefp = cellp->vlServersp;
           sn < AFSMAXCELLHOSTS && serverRefp;
           sn++, serverRefp = serverRefp->next)
     {
@@ -1164,7 +1191,7 @@ GetCellCommon(afs_int32 a_cellnum, char **a_name, serverList *a_hosts)
 }
 
 /* debug interface */
-int 
+int
 SRXAFSCB_GetCellByNum(struct rx_call *callp, afs_int32 a_cellnum,
                       char **a_name, serverList *a_hosts)
 {
@@ -1195,8 +1222,8 @@ SRXAFSCB_GetCellByNum(struct rx_call *callp, afs_int32 a_cellnum,
 }
 
 /* debug interface */
-int 
-SRXAFSCB_TellMeAboutYourself( struct rx_call *callp, 
+int
+SRXAFSCB_TellMeAboutYourself( struct rx_call *callp,
                               struct interfaceAddr *addr,
                               Capabilities * capabilities)
 {
@@ -1241,7 +1268,7 @@ SRXAFSCB_TellMeAboutYourself( struct rx_call *callp,
     for ( i=0; i < cm_noIPAddr; i++ ) {
         addr->addr_in[i] = cm_IPAddr[i];
         addr->subnetmask[i] = cm_SubnetMask[i];
-        addr->mtu[i] = (rx_mtu == -1 || (rx_mtu != -1 && cm_NetMtu[i] < rx_mtu)) ? 
+        addr->mtu[i] = (rx_mtu == -1 || (rx_mtu != -1 && cm_NetMtu[i] < rx_mtu)) ?
             cm_NetMtu[i] : rx_mtu;
     }
     lock_ReleaseRead(&cm_syscfgLock);
@@ -1328,7 +1355,7 @@ int SRXAFSCB_GetServerPrefs(
  *      As advertised.
  *------------------------------------------------------------------------*/
 
-int SRXAFSCB_GetCellServDB(struct rx_call *callp, afs_int32 index, char **a_name, 
+int SRXAFSCB_GetCellServDB(struct rx_call *callp, afs_int32 index, char **a_name,
                            serverList *a_hosts)
 {
     struct rx_connection *connp;
@@ -1386,6 +1413,7 @@ int SRXAFSCB_GetLocalCell(struct rx_call *callp, char **a_name)
     struct rx_peer *peerp;
     unsigned long host = 0;
     unsigned short port = 0;
+    size_t len;
 
     if (cm_shutdown)
         return 1;
@@ -1399,7 +1427,10 @@ int SRXAFSCB_GetLocalCell(struct rx_call *callp, char **a_name)
              ntohl(host), ntohs(port));
 
     if (cm_data.rootCellp) {
-        t_name = strdup(cm_data.rootCellp->name);
+        len = strlen(cm_data.rootCellp->name) + 1;
+        t_name = (char *)xdr_alloc(len);
+        if (t_name)
+            memcpy(t_name, cm_data.rootCellp->name, len);
     } else {
        t_name = (char *)xdr_alloc(1);
        t_name[0] = '\0';
@@ -1444,7 +1475,7 @@ static void afs_MarshallCacheConfig(
     *(ptr++) = config->memCache;
 
 }
+
 
 /*------------------------------------------------------------------------
  * EXPORTED SRXAFSCB_GetCacheConfig
@@ -1532,64 +1563,46 @@ void cm_InitCallback(void)
 int cm_HaveCallback(cm_scache_t *scp)
 {
 #ifdef AFS_FREELANCE_CLIENT
-    // yj: we handle callbacks specially for callbacks on the root directory
-    // Since it's local, we almost always say that we have callback on it
-    // The only time we send back a 0 is if we're need to initialize or
-    // reinitialize the fake directory
-
-    // There are 2 state variables cm_fakeGettingCallback and cm_fakeDirCallback
-    // cm_fakeGettingCallback is 1 if we're in the process of initialization and
-    // hence should return false. it's 0 otherwise
-    // cm_fakeDirCallback is 0 if we haven't loaded the fake directory, it's 1
-    // if the fake directory is loaded and this is the first time cm_HaveCallback
-    // is called since then. We return false in this case to allow cm_GetCallback
-    // to be called because cm_GetCallback has some initialization work to do.
-    // If cm_fakeDirCallback is 2, then it means that the fake directory is in
-    // good shape and we simply return true, provided no change is detected.
-    int fdc, fgc;
-
-    if (cm_freelanceEnabled && 
-         scp->fid.cell==AFS_FAKE_ROOT_CELL_ID && scp->fid.volume==AFS_FAKE_ROOT_VOL_ID) {
-        lock_ObtainMutex(&cm_Freelance_Lock);
-        fdc = cm_fakeDirCallback;
-        fgc = cm_fakeGettingCallback;
-        lock_ReleaseMutex(&cm_Freelance_Lock);
-           
-        if (fdc==1) {  // first call since init
-            return 0;
-        } else if (fdc==2 && !fgc) {   // we're in good shape
-            if (cm_getLocalMountPointChange()) {       // check for changes
-                cm_clearLocalMountPointChange(); // clear the changefile
-                lock_ReleaseWrite(&scp->rw);      // this is re-locked in reInitLocalMountPoints
-                cm_reInitLocalMountPoints();   // start reinit
-                lock_ObtainWrite(&scp->rw);      // now get the lock back 
-                return 0;
-            }
-            return (cm_data.fakeDirVersion == scp->dataVersion);
+    if (cm_freelanceEnabled &&
+        scp->fid.cell==AFS_FAKE_ROOT_CELL_ID &&
+        scp->fid.volume==AFS_FAKE_ROOT_VOL_ID) {
+        if (cm_getLocalMountPointChange()) {
+            cm_clearLocalMountPointChange();
+            lock_ReleaseWrite(&scp->rw);
+            cm_reInitLocalMountPoints();
+            lock_ObtainWrite(&scp->rw);
         }
-        return 0;
+        return (cm_data.fakeDirVersion == scp->dataVersion);
     }
 #endif
-
-    if (scp->cbServerp != NULL) {
-       return 1;
-    } else if (cm_OfflineROIsValid) {
+    if (cm_readonlyVolumeVersioning &&
+        (scp->flags & CM_SCACHEFLAG_PURERO)) {
         cm_volume_t *volp = cm_GetVolumeByFID(&scp->fid);
         if (volp) {
-            switch (cm_GetVolumeStatus(volp, scp->fid.volume)) {
-            case vl_offline:
-            case vl_alldown:
-            case vl_unknown:
-                cm_PutVolume(volp);
-                return 1;
-            default:
-                cm_PutVolume(volp);
-                return 0;
+            int haveCB = 0;
+            if (cm_OfflineROIsValid) {
+                switch (cm_GetVolumeStatus(volp, scp->fid.volume)) {
+                case vl_offline:
+                case vl_alldown:
+                case vl_unknown:
+                    haveCB = 1;
+                    break;
+                }
             }
+            if (cm_readonlyVolumeVersioning &&
+                !haveCB &&
+                volp->creationDateRO == scp->volumeCreationDate &&
+                volp->cbServerpRO != NULL) {
+                haveCB = 1;
+            }
+            cm_PutVolume(volp);
+            return haveCB;
         }
-        return 1;
     }
-    return 0;
+    if (scp->cbServerp != NULL)
+       return 1;
+    else
+        return 0;
 }
 
 /* need to detect a broken callback that races with our obtaining a callback.
@@ -1604,7 +1617,7 @@ int cm_HaveCallback(cm_scache_t *scp)
  * When we're back from the call, we look at all of the callback revokes with
  * counter numbers greater than the one we recorded in our caller's structure,
  * and replay those that are higher than when we started the call.
- * 
+ *
  * We free all the structures in the queue when the count of the # of outstanding
  * callback-granting calls drops to zero.
  *
@@ -1626,52 +1639,40 @@ void cm_StartCallbackGrantingCall(cm_scache_t *scp, cm_callbackRequest_t *cbrp)
  *
  * Called with scp write locked, so we can discard the callbacks easily with
  * this locking hierarchy.
+ *
+ * Performs cleanup of the stack allocated cm_callbackRequest_t object.
+ *
+ * Returns 0 on success and non-zero if the callback lost a race.
  */
-void cm_EndCallbackGrantingCall(cm_scache_t *scp, cm_callbackRequest_t *cbrp,
-                                AFSCallBack *cbp, long flags)
+int
+cm_EndCallbackGrantingCall(cm_scache_t *scp, cm_callbackRequest_t *cbrp,
+                           AFSCallBack *cbp, AFSVolSync *volSyncp, long flags)
 {
     cm_racingRevokes_t *revp;          /* where we are */
     cm_racingRevokes_t *nrevp;         /* where we'll be next */
-    int freeFlag;
+    int freeRacingRevokes;
+    int freeServer;
     cm_server_t * serverp = NULL;
-    int discardScp = 0, discardVolCB = 0;
+    int lostRace = 0;
 
     lock_ObtainWrite(&cm_callbackLock);
     if (flags & CM_CALLBACK_MAINTAINCOUNT) {
-        osi_assertx(cm_activeCallbackGrantingCalls > 0, 
+        osi_assertx(cm_activeCallbackGrantingCalls > 0,
                     "CM_CALLBACK_MAINTAINCOUNT && cm_activeCallbackGrantingCalls == 0");
+        freeServer = 0;
     }
     else {
         osi_assertx(cm_activeCallbackGrantingCalls-- > 0,
                     "!CM_CALLBACK_MAINTAINCOUNT && cm_activeCallbackGrantingCalls == 0");
+        freeServer = 1;
     }
-    if (cm_activeCallbackGrantingCalls == 0) 
-        freeFlag = 1;
-    else 
-        freeFlag = 0;
-
-    /* record the callback; we'll clear it below if we really lose it */
-    if (cbrp) {
-       if (scp) {
-            if (!cm_ServerEqual(scp->cbServerp, cbrp->serverp)) {
-                serverp = scp->cbServerp;
-                if (!freeFlag)
-                    cm_GetServer(cbrp->serverp);
-                scp->cbServerp = cbrp->serverp;
-            } else {
-                if (freeFlag)
-                    serverp = cbrp->serverp;
-            }
-            scp->cbExpires = cbrp->startTime + cbp->ExpirationTime;
-        } else {
-            if (freeFlag)
-                serverp = cbrp->serverp;
-        }
-        if (freeFlag)
-            cbrp->serverp = NULL;
-    }
+    if (cm_activeCallbackGrantingCalls == 0)
+        freeRacingRevokes = 1;
+    else
+        freeRacingRevokes = 0;
 
-    /* a callback was actually revoked during our granting call, so
+    /*
+     * a callback was revoked during our granting call, so
      * run down the list of revoked fids, looking for ours.
      * If activeCallbackGrantingCalls is zero, free the elements, too.
      *
@@ -1692,7 +1693,7 @@ void cm_EndCallbackGrantingCall(cm_scache_t *scp, cm_callbackRequest_t *cbrp,
                   ((revp->flags & CM_RACINGFLAG_CANCELVOL) &&
                     scp->fid.volume == revp->fid.volume)
                   ||
-                  ((revp->flags & CM_RACINGFLAG_CANCELALL) && 
+                  ((revp->flags & CM_RACINGFLAG_CANCELALL) &&
                    (revp->fid.cell == 0 || scp->fid.cell == revp->fid.cell)))) {
             /* this one matches */
             osi_Log4(afsd_logp,
@@ -1700,52 +1701,80 @@ void cm_EndCallbackGrantingCall(cm_scache_t *scp, cm_callbackRequest_t *cbrp,
                       scp,
                       cbrp->callbackCount, revp->callbackCount,
                       cm_callbackCount);
-            discardScp = 1;
-            if ((scp->flags & CM_SCACHEFLAG_PURERO) && 
-                 (revp->flags & CM_RACINGFLAG_ALL)) {
-                cm_volume_t *volp = cm_GetVolumeByFID(&scp->fid);
-                if (volp) {
-                    volp->cbExpiresRO = 0;
-                    cm_PutVolume(volp);
-                }
-            }
+            lostRace = 1;
+            if ((scp->flags & CM_SCACHEFLAG_PURERO) &&
+                 (revp->flags & CM_RACINGFLAG_ALL))
+                cm_callbackDiscardROVolumeByFID(&scp->fid);
         }
-        if (freeFlag) 
+        if (freeRacingRevokes)
             free(revp);
     }
 
     /* if we freed the list, zap the pointer to it */
-    if (freeFlag) 
+    if (freeRacingRevokes)
         cm_racingRevokesp = NULL;
-
     lock_ReleaseWrite(&cm_callbackLock);
 
-    if ( discardScp ) {
-        cm_DiscardSCache(scp);
-        lock_ReleaseWrite(&scp->rw);
-        cm_CallbackNotifyChange(scp);
-        lock_ObtainWrite(&scp->rw);
-    } else {
-        if (scp && scp->flags & CM_SCACHEFLAG_PURERO) {
+    /* record the callback if we didn't lose it */
+    if (!lostRace && scp) {
+        if (cbrp) {
+            if (!cm_ServerEqual(scp->cbServerp, cbrp->serverp)) {
+                serverp = scp->cbServerp;
+                scp->cbServerp = cbrp->serverp;
+                if (freeServer) {
+                    cbrp->serverp = NULL;
+                } else {
+                    cm_GetServer(scp->cbServerp);
+                }
+            } else if (freeServer) {
+                serverp = cbrp->serverp;
+                cbrp->serverp = NULL;
+            }
+            scp->cbExpires = cbrp->startTime + cbp->ExpirationTime;
+            scp->cbIssued = time(NULL);
+        }
+
+        if (scp->flags & CM_SCACHEFLAG_PURERO) {
             cm_volume_t * volp = cm_GetVolumeByFID(&scp->fid);
             if (volp) {
-                volp->cbExpiresRO = scp->cbExpires;
-                if (volp->cbServerpRO != scp->cbServerp) {
-                    if (volp->cbServerpRO)
-                        cm_PutServer(volp->cbServerpRO);
-                    cm_GetServer(scp->cbServerp);
-                    volp->cbServerpRO = scp->cbServerp;
+                if (volSyncp) {
+                    lock_ObtainWrite(&cm_scacheLock);
+                    volp->cbExpiresRO = scp->cbExpires;
+                    volp->cbIssuedRO = scp->cbIssued;
+                    volp->creationDateRO = volSyncp->spare1;
+                    if (volp->cbServerpRO != scp->cbServerp) {
+                        if (volp->cbServerpRO)
+                            cm_PutServer(volp->cbServerpRO);
+                        cm_GetServer(scp->cbServerp);
+                        volp->cbServerpRO = scp->cbServerp;
+                    }
+                    lock_ReleaseWrite(&cm_scacheLock);
                 }
                 cm_PutVolume(volp);
             }
         }
     }
+    else
+    {
+        /*
+         * No need to discard the cm_scache object or notify the
+         * afs redirector or smb client about a change if we did lose
+         * the race.  That was done when the callback was processed in
+         * cm_RevokeCallback().
+         */
+        if (cbrp && freeServer) {
+            serverp = cbrp->serverp;
+            cbrp->serverp = NULL;
+        }
+    }
 
     if ( serverp ) {
         lock_ObtainWrite(&cm_serverLock);
         cm_FreeServer(serverp);
         lock_ReleaseWrite(&cm_serverLock);
     }
+
+    return lostRace;
 }
 
 /* if flags is 1, we want to force the code to make one call, anyway.
@@ -1766,13 +1795,15 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
     struct rx_connection * rxconnp = NULL;
     int syncop_done = 0;
 
-    osi_Log4(afsd_logp, "GetCallback scp 0x%p cell %d vol %d flags %lX", 
+    memset(&volSync, 0, sizeof(volSync));
+
+    osi_Log4(afsd_logp, "GetCallback scp 0x%p cell %d vol %d flags %lX",
              scp, scp->fid.cell, scp->fid.volume, flags);
 
 #ifdef AFS_FREELANCE_CLIENT
     // The case where a callback is needed on /afs is handled
     // specially. We need to fetch the status by calling
-    // cm_MergeStatus and mark that cm_fakeDirCallback is 2
+    // cm_MergeStatus
     if (cm_freelanceEnabled &&
         (scp->fid.cell==AFS_FAKE_ROOT_CELL_ID &&
          scp->fid.volume==AFS_FAKE_ROOT_VOL_ID)) {
@@ -1783,34 +1814,18 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
             goto done;
         syncop_done = 1;
 
-        if (cm_fakeDirCallback != 2) {
-            // Start by indicating that we're in the process
-            // of fetching the callback
-            lock_ObtainMutex(&cm_Freelance_Lock);
-            osi_Log0(afsd_logp,"GetCallback Freelance fakeGettingCallback=1");
-            cm_fakeGettingCallback = 1;
-            lock_ReleaseMutex(&cm_Freelance_Lock);
-
+        if (scp->dataVersion != cm_data.fakeDirVersion) {
             memset(&afsStatus, 0, sizeof(afsStatus));
             memset(&volSync, 0, sizeof(volSync));
+            InterlockedIncrement(&scp->activeRPCs);
 
-            // Fetch the status info 
+            // Fetch the status info
             cm_MergeStatus(NULL, scp, &afsStatus, &volSync, userp, reqp, 0);
-
-            // Indicate that the callback is not done
-            lock_ObtainMutex(&cm_Freelance_Lock);
-            osi_Log0(afsd_logp,"GetCallback Freelance fakeDirCallback=2");
-            cm_fakeDirCallback = 2;
-
-            // Indicate that we're no longer fetching the callback
-            osi_Log0(afsd_logp,"GetCallback Freelance fakeGettingCallback=0");
-            cm_fakeGettingCallback = 0;
-            lock_ReleaseMutex(&cm_Freelance_Lock);
         }
         goto done;
     }
 #endif /* AFS_FREELANCE_CLIENT */
-       
+
     mustCall = (flags & 1);
     cm_AFSFidFromFid(&tfid, &scp->fid);
     while (1) {
@@ -1822,7 +1837,7 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
 
         /* otherwise, we have to make an RPC to get the status */
        if (!syncop_done) {
-           code = cm_SyncOp(scp, NULL, userp, reqp, 0, 
+           code = cm_SyncOp(scp, NULL, userp, reqp, 0,
                             CM_SCACHESYNC_FETCHSTATUS | CM_SCACHESYNC_GETCALLBACK);
            if (code)
                break;
@@ -1831,13 +1846,14 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
         cm_StartCallbackGrantingCall(scp, &cbr);
         sfid = scp->fid;
         lock_ReleaseWrite(&scp->rw);
-               
+
         /* now make the RPC */
-        osi_Log4(afsd_logp, "CALL FetchStatus scp 0x%p vol %u vn %u uniq %u", 
+        InterlockedIncrement(&scp->activeRPCs);
+        osi_Log4(afsd_logp, "CALL FetchStatus scp 0x%p vol %u vn %u uniq %u",
                  scp, sfid.volume, sfid.vnode, sfid.unique);
         do {
             code = cm_ConnFromFID(&sfid, userp, reqp, &connp);
-            if (code) 
+            if (code)
                 continue;
 
             rxconnp = cm_GetRxConn(connp);
@@ -1845,22 +1861,24 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
                                      &afsStatus, &callback, &volSync);
             rx_PutConnection(rxconnp);
 
-        } while (cm_Analyze(connp, userp, reqp, &sfid, &volSync, NULL,
+        } while (cm_Analyze(connp, userp, reqp, &sfid, 0, &volSync, NULL,
                             &cbr, code));
         code = cm_MapRPCError(code, reqp);
         if (code)
-            osi_Log4(afsd_logp, "CALL FetchStatus FAILURE code 0x%x scp 0x%p vol %u vn %u", 
+            osi_Log4(afsd_logp, "CALL FetchStatus FAILURE code 0x%x scp 0x%p vol %u vn %u",
                      code, scp, scp->fid.volume, scp->fid.vnode);
         else
-            osi_Log4(afsd_logp, "CALL FetchStatus SUCCESS scp 0x%p vol %u vn %u uniq %u", 
+            osi_Log4(afsd_logp, "CALL FetchStatus SUCCESS scp 0x%p vol %u vn %u uniq %u",
                      scp, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
 
         lock_ObtainWrite(&scp->rw);
         if (code == 0) {
-            cm_EndCallbackGrantingCall(scp, &cbr, &callback, 0);
-            cm_MergeStatus(NULL, scp, &afsStatus, &volSync, userp, reqp, 0);
+            int lostRace = cm_EndCallbackGrantingCall(scp, &cbr, &callback, &volSync, 0);
+            if (!lostRace)
+                cm_MergeStatus(NULL, scp, &afsStatus, &volSync, userp, reqp, 0);
         } else {
-            cm_EndCallbackGrantingCall(NULL, &cbr, NULL, 0);
+            cm_EndCallbackGrantingCall(NULL, &cbr, NULL, NULL, 0);
+            InterlockedDecrement(&scp->activeRPCs);
         }
 
         /* if we got an error, return to caller */
@@ -1871,13 +1889,13 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
   done:
     if (syncop_done)
        cm_SyncOpDone(scp, NULL, CM_SCACHESYNC_FETCHSTATUS | CM_SCACHESYNC_GETCALLBACK);
-    
+
     if (code) {
        osi_Log2(afsd_logp, "GetCallback Failed code 0x%x scp 0x%p -->",code, scp);
        osi_Log4(afsd_logp, "            cell %u vol %u vn %u uniq %u",
                 scp->fid.cell, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
     } else {
-       osi_Log3(afsd_logp, "GetCallback Complete scp 0x%p cell %d vol %d", 
+       osi_Log3(afsd_logp, "GetCallback Complete scp 0x%p cell %d vol %d",
                  scp, scp->fid.cell, scp->fid.volume);
     }
 
@@ -1885,43 +1903,74 @@ long cm_GetCallback(cm_scache_t *scp, struct cm_user *userp,
 }
 
 
-/* called with cm_scacheLock held */
-long cm_CBServersUp(cm_scache_t *scp, time_t * downTime)
+/*
+ * cm_CBServersDownTime() returns 1 if the downTime parameter is valid.
+ *
+ * Servers with multiple interfaces have multiple cm_server_t objects
+ * which share the same UUID.  If one interface is down but others are up,
+ * the server should not be considered down.  The returned downTime should
+ * be the largest non-zero value if down or zero if up.  If the cbServerp
+ * is down, it is updated to refer to an interface that is up (if one exists).
+ *
+ * called with cm_scacheLock held
+ */
+static long
+cm_CBServersDownTime(cm_scache_t *scp, cm_volume_t *volp, time_t * pdownTime)
 {
     cm_vol_state_t *statep;
-    cm_volume_t * volp;
-    afs_uint32 volID = scp->fid.volume;
     cm_serverRef_t *tsrp;
-    int found;
+    int alldown = 1;
+    time_t downTime = 0;
+    cm_server_t * upserver = NULL;
+    cm_server_t * downserver;
 
-    *downTime = 0;
+    *pdownTime = 0;
 
     if (scp->cbServerp == NULL)
         return 1;
 
-    volp = cm_GetVolumeByFID(&scp->fid);
-    if (!volp)
+    if (!(scp->cbServerp->flags & CM_SERVERFLAG_DOWN))
         return 1;
 
-    statep = cm_VolumeStateByID(volp, volID);
-    cm_PutVolume(volp);
-    if (statep->state == vl_online)
-        return 1;
+    statep = cm_VolumeStateByID(volp, scp->fid.volume);
+    if (statep) {
+        for (tsrp = statep->serversp; tsrp; tsrp=tsrp->next) {
+            if (tsrp->status == srv_deleted)
+                continue;
 
-    for (found = 0,tsrp = statep->serversp; tsrp; tsrp=tsrp->next) {
-        if (tsrp->status == srv_deleted)
-            continue;
-        if (cm_ServerEqual(tsrp->server, scp->cbServerp))
-            found = 1;
-        if (tsrp->server->downTime > *downTime)
-            *downTime = tsrp->server->downTime;
+            if (!cm_ServerEqual(tsrp->server, scp->cbServerp))
+                continue;
+
+            if (!(tsrp->server->flags & CM_SERVERFLAG_DOWN)) {
+                alldown = 0;
+                if (!upserver) {
+                    upserver = tsrp->server;
+                    cm_GetServer(upserver);
+                }
+            }
+
+            if (tsrp->server->downTime > downTime)
+                downTime = tsrp->server->downTime;
+        }
+    } else {
+        downTime = scp->cbServerp->downTime;
     }
 
     /* if the cbServerp does not match the current volume server list
-     * we report the callback server as up so the callback can be 
+     * we report the callback server as up so the callback can be
      * expired.
      */
-    return(found ? 0 : 1);
+
+    if (alldown) {
+        *pdownTime = downTime;
+    } else {
+        lock_ObtainWrite(&scp->rw);
+        downserver = scp->cbServerp;
+        scp->cbServerp = upserver;
+        lock_ReleaseWrite(&scp->rw);
+        cm_PutServer(downserver);
+    }
+    return 1;
 }
 
 /* called periodically by cm_daemon to shut down use of expired callbacks */
@@ -1929,48 +1978,87 @@ void cm_CheckCBExpiration(void)
 {
     afs_uint32 i;
     cm_scache_t *scp;
+    cm_volume_t *volp;
+    enum volstatus volstate;
     time_t now, downTime;
-        
+
     osi_Log0(afsd_logp, "CheckCBExpiration");
 
     now = time(NULL);
     lock_ObtainWrite(&cm_scacheLock);
     for (i=0; i<cm_data.scacheHashTableSize; i++) {
         for (scp = cm_data.scacheHashTablep[i]; scp; scp=scp->nextp) {
+            volp = NULL;
+            cm_HoldSCacheNoLock(scp);
+            lock_ReleaseWrite(&cm_scacheLock);
+
+            /*
+             * If this is not a PURERO object and there is no callback
+             * or it hasn't expired, there is nothing to do
+             */
+            if (!(scp->flags & CM_SCACHEFLAG_PURERO) &&
+                (scp->cbServerp == NULL || scp->cbExpires == 0 || now < scp->cbExpires))
+                goto scp_complete;
+
+            /*
+             * Determine the volume state and update the callback info
+             * to the latest if it is a PURERO object.
+             */
+            volp = cm_GetVolumeByFID(&scp->fid);
+            volstate = vl_unknown;
             downTime = 0;
-            if (scp->flags & CM_SCACHEFLAG_PURERO) {
-                cm_volume_t *volp = cm_GetVolumeByFID(&scp->fid);
-                if (volp) {
-                    if (volp->cbExpiresRO > scp->cbExpires &&
-                        scp->cbExpires > 0) 
-                    {
-                        scp->cbExpires = volp->cbExpiresRO;
-                        if (volp->cbServerpRO != scp->cbServerp) {
-                            if (scp->cbServerp)
-                               cm_PutServer(scp->cbServerp);
-                           cm_GetServer(volp->cbServerpRO);
-                           scp->cbServerp = volp->cbServerpRO;
-                        }
-                    }        
-                    cm_PutVolume(volp);
+            if (volp) {
+                if ((scp->flags & CM_SCACHEFLAG_PURERO) &&
+                    volp->cbExpiresRO > scp->cbExpires && scp->cbExpires > 0)
+                {
+                    lock_ObtainWrite(&scp->rw);
+                    scp->cbExpires = volp->cbExpiresRO;
+                    scp->cbIssued = volp->cbIssuedRO;
+                    if (volp->cbServerpRO != scp->cbServerp) {
+                        if (scp->cbServerp)
+                            cm_PutServer(scp->cbServerp);
+                        cm_GetServer(volp->cbServerpRO);
+                        scp->cbServerp = volp->cbServerpRO;
+                    }
+                    lock_ReleaseWrite(&scp->rw);
                 }
+                volstate = cm_GetVolumeStatus(volp, scp->fid.volume);
             }
-            if (scp->cbServerp && scp->cbExpires > 0 && now > scp->cbExpires && 
-                 (cm_CBServersUp(scp, &downTime) || downTime == 0 || downTime >= scp->cbExpires)) 
-            {
-                cm_HoldSCacheNoLock(scp);
-                lock_ReleaseWrite(&cm_scacheLock);
-                
-                osi_Log4(afsd_logp, "Callback Expiration Discarding SCache scp 0x%p vol %u vn %u uniq %u",
-                          scp, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
-                lock_ObtainWrite(&scp->rw);
-                cm_DiscardSCache(scp);
-                lock_ReleaseWrite(&scp->rw);
-                cm_CallbackNotifyChange(scp);
 
-                lock_ObtainWrite(&cm_scacheLock);
-                cm_ReleaseSCacheNoLock(scp);
-            }
+            /* If there is no callback or it hasn't expired, there is nothing to do */
+            if (scp->cbServerp == NULL || scp->cbExpires == 0 || now < scp->cbExpires)
+                goto scp_complete;
+
+            /* If the volume is known not to be online, do not expire the callback */
+            if (volstate != vl_online)
+                goto scp_complete;
+
+            /*
+             * If all the servers are down and the callback expired after the
+             * issuing server went down, do not expire the callback
+             */
+            if (cm_CBServersDownTime(scp, volp, &downTime) && downTime && downTime < scp->cbExpires)
+                goto scp_complete;
+
+            /* The callback has expired, discard the status info */
+            osi_Log4(afsd_logp, "Callback Expiration Discarding SCache scp 0x%p vol %u vn %u uniq %u",
+                     scp, scp->fid.volume, scp->fid.vnode, scp->fid.unique);
+            lock_ObtainWrite(&scp->rw);
+            cm_DiscardSCache(scp);
+            lock_ReleaseWrite(&scp->rw);
+
+            if (RDR_Initialized)
+                RDR_InvalidateObject(scp->fid.cell, scp->fid.volume, scp->fid.vnode, scp->fid.unique,
+                                     scp->fid.hash, scp->fileType, AFS_INVALIDATE_EXPIRED);
+
+            cm_CallbackNotifyChange(scp);
+
+          scp_complete:
+            if (volp)
+                cm_PutVolume(volp);
+
+            lock_ObtainWrite(&cm_scacheLock);
+            cm_ReleaseSCacheNoLock(scp);
         }
     }
     lock_ReleaseWrite(&cm_scacheLock);
@@ -1979,16 +2067,16 @@ void cm_CheckCBExpiration(void)
 }
 
 
-void 
+void
 cm_GiveUpAllCallbacks(cm_server_t *tsp, afs_int32 markDown)
 {
     long code;
     cm_conn_t *connp;
     struct rx_connection * rxconnp;
 
-    if ((tsp->type == CM_SERVER_FILE) && !(tsp->flags & CM_SERVERFLAG_DOWN)) 
+    if ((tsp->type == CM_SERVER_FILE) && !(tsp->flags & CM_SERVERFLAG_DOWN))
     {
-        code = cm_ConnByServer(tsp, cm_rootUserp, &connp);
+        code = cm_ConnByServer(tsp, cm_rootUserp, FALSE, &connp);
         if (code == 0) {
             rxconnp = cm_GetRxConn(connp);
             rx_SetConnDeadTime(rxconnp, 10);
@@ -2006,7 +2094,7 @@ cm_GiveUpAllCallbacks(cm_server_t *tsp, afs_int32 markDown)
 
             lock_ObtainMutex(&tsp->mx);
             if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
-                tsp->flags |= CM_SERVERFLAG_DOWN;
+                _InterlockedOr(&tsp->flags, CM_SERVERFLAG_DOWN);
                 tsp->downTime = time(NULL);
             }
             /* Now update the volume status */
@@ -2020,10 +2108,10 @@ cm_GiveUpAllCallbacks(cm_server_t *tsp, afs_int32 markDown)
                         code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
                                                  &req, CM_GETVOL_FLAG_NO_LRU_UPDATE | CM_GETVOL_FLAG_NO_RESET, &volp);
                         lock_ObtainMutex(&tsp->mx);
-                        if (code == 0) {    
+                        if (code == 0) {
                             cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
                             cm_PutVolume(volp);
-                        }       
+                        }
                     }
                 }
             }
@@ -2040,15 +2128,121 @@ cm_GiveUpAllCallbacksAllServers(afs_int32 markDown)
     if (!cm_giveUpAllCBs)
         return;
 
-    lock_ObtainWrite(&cm_serverLock);
+    lock_ObtainRead(&cm_serverLock);
     for (tsp = cm_allServersp; tsp; tsp = tsp->allNextp) {
         cm_GetServerNoLock(tsp);
-        lock_ReleaseWrite(&cm_serverLock);
+        lock_ReleaseRead(&cm_serverLock);
         cm_GiveUpAllCallbacks(tsp, markDown);
-        lock_ObtainWrite(&cm_serverLock);
+        lock_ObtainRead(&cm_serverLock);
         cm_PutServerNoLock(tsp);
     }
-    lock_ReleaseWrite(&cm_serverLock);
+    lock_ReleaseRead(&cm_serverLock);
+}
+
+void
+cm_GiveUpAllCallbacksAllServersMulti(afs_int32 markDown)
+{
+    long code;
+    cm_conn_t **conns = NULL;
+    struct rx_connection **rxconns = NULL;
+    afs_int32 i, nconns = 0, maxconns;
+    cm_server_t ** serversp, *tsp;
+    afs_int32 *results;
+    time_t start, *deltas;
+
+    maxconns = cm_numFileServers;
+    if (maxconns == 0)
+        return;
+
+    conns = (cm_conn_t **)malloc(maxconns * sizeof(cm_conn_t *));
+    rxconns = (struct rx_connection **)malloc(maxconns * sizeof(struct rx_connection *));
+    deltas = (time_t *)malloc(maxconns * sizeof (time_t));
+    results = (afs_int32 *)malloc(maxconns * sizeof (afs_int32));
+    serversp = (cm_server_t **)malloc(maxconns * sizeof(cm_server_t *));
+
+    lock_ObtainRead(&cm_serverLock);
+    for (nconns=0, tsp = cm_allServersp; tsp && nconns < maxconns; tsp = tsp->allNextp) {
+        if (tsp->type != CM_SERVER_FILE ||
+            (tsp->flags & CM_SERVERFLAG_DOWN) ||
+            tsp->cellp == NULL          /* SetPrefs only */)
+            continue;
+
+        cm_GetServerNoLock(tsp);
+        lock_ReleaseRead(&cm_serverLock);
+
+        serversp[nconns] = tsp;
+        code = cm_ConnByServer(tsp, cm_rootUserp, FALSE, &conns[nconns]);
+        if (code) {
+            lock_ObtainRead(&cm_serverLock);
+            cm_PutServerNoLock(tsp);
+            continue;
+        }
+        lock_ObtainRead(&cm_serverLock);
+        rxconns[nconns] = cm_GetRxConn(conns[nconns]);
+        rx_SetConnDeadTime(rxconns[nconns], 10);
+
+        nconns++;
+    }
+    lock_ReleaseRead(&cm_serverLock);
+
+    if (nconns) {
+        /* Perform the multi call */
+        start = time(NULL);
+        multi_Rx(rxconns,nconns)
+        {
+            multi_RXAFS_GiveUpAllCallBacks();
+            results[multi_i]=multi_error;
+        } multi_End;
+    }
+
+    /* Process results of servers that support RXAFS_GetCapabilities */
+    for (i=0; i<nconns; i++) {
+        rx_SetConnDeadTime(rxconns[i], ConnDeadtimeout);
+        rx_PutConnection(rxconns[i]);
+        cm_PutConn(conns[i]);
+
+        tsp = serversp[i];
+        cm_GCConnections(tsp);
+
+        if (markDown) {
+            cm_server_vols_t * tsrvp;
+            cm_volume_t * volp;
+            int i;
+
+            cm_ForceNewConnections(tsp);
+
+            lock_ObtainMutex(&tsp->mx);
+            if (!(tsp->flags & CM_SERVERFLAG_DOWN)) {
+                _InterlockedOr(&tsp->flags, CM_SERVERFLAG_DOWN);
+                tsp->downTime = time(NULL);
+            }
+            /* Now update the volume status */
+            for (tsrvp = tsp->vols; tsrvp; tsrvp = tsrvp->nextp) {
+                for (i=0; i<NUM_SERVER_VOLS; i++) {
+                    if (tsrvp->ids[i] != 0) {
+                        cm_req_t req;
+
+                        cm_InitReq(&req);
+                        lock_ReleaseMutex(&tsp->mx);
+                        code = cm_FindVolumeByID(tsp->cellp, tsrvp->ids[i], cm_rootUserp,
+                                                 &req, CM_GETVOL_FLAG_NO_LRU_UPDATE | CM_GETVOL_FLAG_NO_RESET, &volp);
+                        lock_ObtainMutex(&tsp->mx);
+                        if (code == 0) {
+                            cm_UpdateVolumeStatus(volp, tsrvp->ids[i]);
+                            cm_PutVolume(volp);
+                        }
+                    }
+                }
+            }
+            lock_ReleaseMutex(&tsp->mx);
+        }
+    }
+
+    free(conns);
+    free(rxconns);
+    free(deltas);
+    free(results);
+    free(serversp);
 }