afs: do not allow two shutdown sequences in parallel

Often, ‘afsd -shutdown’ is called right after ‘umount’.
Both commands hold the glock before calling ‘afs_shutdown’.
However, one of the functions called by 'afs_shutdown', namely,
‘afs_FlushVCBs’, might drop the glock when the global
'afs_shuttingdown' is still equal to 0. As a result, a scenario
with two shutdown sequences proceeding in parallel is possible.

To fix the problem, the global ‘afs_shuttingdown’ is used as an
enumerated type to make sure that the second thread will not run
‘afs_shutdown’ while the first one is stuck inside ‘afs_FlushVCBs’.

Change-Id: Iffa89d82278b0df5fb90fc35608af66d8e8db29e
Reviewed-on: http://gerrit.openafs.org/12016
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Chas Williams <3chas3@gmail.com>
Reviewed-by: Michael Meffie <mmeffie@sinenomine.net>
Reviewed-by: Benjamin Kaduk <kaduk@mit.edu>

diff --git a/src/afs/AIX/osi_file.c b/src/afs/AIX/osi_file.c
index 9e70da4..4bd3371 100644 (file)
--- a/src/afs/AIX/osi_file.c
+++ b/src/afs/AIX/osi_file.c
@@ -161,7 +161,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/DARWIN/osi_file.c b/src/afs/DARWIN/osi_file.c
index b619910..a83dbf2 100644 (file)
--- a/src/afs/DARWIN/osi_file.c
+++ b/src/afs/DARWIN/osi_file.c
@@ -318,7 +318,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/DARWIN/osi_module.c b/src/afs/DARWIN/osi_module.c
index 9029253..046870a 100644 (file)
--- a/src/afs/DARWIN/osi_module.c
+++ b/src/afs/DARWIN/osi_module.c
@@ -115,7 +115,7 @@ afs_modunload(struct kmod_info * kmod_info, void *data)
 {
     if (afs_globalVFS)
        return KERN_FAILURE;
-    if ((afs_initState != 0) || (afs_shuttingdown))
+    if ((afs_initState != 0) || (afs_shuttingdown != AFS_RUNNING))
        return KERN_FAILURE;
 #ifdef AFS_DARWIN80_ENV
     if (vfs_fsremove(afs_vfstable))

diff --git a/src/afs/FBSD/osi_file.c b/src/afs/FBSD/osi_file.c
index 096e472..1cfc5c8 100644 (file)
--- a/src/afs/FBSD/osi_file.c
+++ b/src/afs/FBSD/osi_file.c
@@ -171,7 +171,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/HPUX/osi_file.c b/src/afs/HPUX/osi_file.c
index 44ed271..a562fd4 100644 (file)
--- a/src/afs/HPUX/osi_file.c
+++ b/src/afs/HPUX/osi_file.c
@@ -147,7 +147,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/HPUX/osi_vnodeops.c b/src/afs/HPUX/osi_vnodeops.c
index 839d76e..bce2441 100644 (file)
--- a/src/afs/HPUX/osi_vnodeops.c
+++ b/src/afs/HPUX/osi_vnodeops.c
@@ -297,7 +297,7 @@ afs_inactive(avc, acred)
     struct vnode *vp = AFSTOV(avc);
     ulong_t context;
     lock_t *sv_lock;
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
 
     /*

diff --git a/src/afs/IRIX/osi_file.c b/src/afs/IRIX/osi_file.c
index bdb458f..92a366c 100644 (file)
--- a/src/afs/IRIX/osi_file.c
+++ b/src/afs/IRIX/osi_file.c
@@ -141,7 +141,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX/osi_file.c b/src/afs/LINUX/osi_file.c
index c050715..c125c1c 100644 (file)
--- a/src/afs/LINUX/osi_file.c
+++ b/src/afs/LINUX/osi_file.c
@@ -218,7 +218,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;
@@ -258,7 +258,7 @@ afs_osi_Write(struct osi_file *afile, afs_int32 offset, void *aptr,
     AFS_STATCNT(osi_Write);
 
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("afs_osi_Write called with null param");
        else
            return -EIO;

diff --git a/src/afs/LINUX/osi_vnodeops.c b/src/afs/LINUX/osi_vnodeops.c
index b2e6dd7..6b49b5e 100644 (file)
--- a/src/afs/LINUX/osi_vnodeops.c
+++ b/src/afs/LINUX/osi_vnodeops.c
@@ -945,7 +945,7 @@ afs_linux_revalidate(struct dentry *dp)
     cred_t *credp;
     int code;
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     AFS_GLOCK();

diff --git a/src/afs/NBSD/osi_file.c b/src/afs/NBSD/osi_file.c
index dd7811a..dfacb69 100644 (file)
--- a/src/afs/NBSD/osi_file.c
+++ b/src/afs/NBSD/osi_file.c
@@ -154,7 +154,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr, afs_int32 asize)
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/OBSD/osi_file.c b/src/afs/OBSD/osi_file.c
index c19d86b..7929141 100644 (file)
--- a/src/afs/OBSD/osi_file.c
+++ b/src/afs/OBSD/osi_file.c
@@ -143,7 +143,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr, afs_int32 asize)
      * down. No point in crashing when we are already shutting down
      */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/SOLARIS/osi_file.c b/src/afs/SOLARIS/osi_file.c
index b4380f3..a3387ec 100644 (file)
--- a/src/afs/SOLARIS/osi_file.c
+++ b/src/afs/SOLARIS/osi_file.c
@@ -354,7 +354,7 @@ afs_osi_Read(struct osi_file *afile, int offset, void *aptr,
       * down. No point in crashing when we are already shutting down
       */
     if (!afile) {
-       if (!afs_shuttingdown)
+       if (afs_shuttingdown == AFS_RUNNING)
            osi_Panic("osi_Read called with null param");
        else
            return -EIO;

diff --git a/src/afs/SOLARIS/osi_vnodeops.c b/src/afs/SOLARIS/osi_vnodeops.c
index d19bcf6..69dd985 100644 (file)
--- a/src/afs/SOLARIS/osi_vnodeops.c
+++ b/src/afs/SOLARIS/osi_vnodeops.c
@@ -1659,7 +1659,7 @@ int
 afs_inactive(struct vcache *avc, afs_ucred_t *acred)
 {
     struct vnode *vp = AFSTOV(avc);
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return 0;
 
     /*

diff --git a/src/afs/UKERNEL/osi_vnodeops.c b/src/afs/UKERNEL/osi_vnodeops.c
index 290d46d..c78fe27 100644 (file)
--- a/src/afs/UKERNEL/osi_vnodeops.c
+++ b/src/afs/UKERNEL/osi_vnodeops.c
@@ -33,7 +33,7 @@ afs_vrdwr(struct usr_vnode *avc, struct usr_uio *uio, int rw, int io,
 int
 afs_inactive(struct vcache *avc, afs_ucred_t *acred)
 {
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return 0;
 
     usr_assert(avc->vrefCount == 0);

diff --git a/src/afs/VNOPS/afs_vnop_attrs.c b/src/afs/VNOPS/afs_vnop_attrs.c
index 7e580c5..cc1fd32 100644 (file)
--- a/src/afs/VNOPS/afs_vnop_attrs.c
+++ b/src/afs/VNOPS/afs_vnop_attrs.c
@@ -241,7 +241,7 @@ afs_getattr(OSI_VC_DECL(avc), struct vattr *attrs, afs_ucred_t *acred)
 
     AFS_DISCON_LOCK();
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown != AFS_RUNNING) {
        AFS_DISCON_UNLOCK();
        return EIO;
     }

diff --git a/src/afs/VNOPS/afs_vnop_fid.c b/src/afs/VNOPS/afs_vnop_fid.c
index cdd147e..d10bcac 100644 (file)
--- a/src/afs/VNOPS/afs_vnop_fid.c
+++ b/src/afs/VNOPS/afs_vnop_fid.c
@@ -83,7 +83,7 @@ afs_fid(OSI_VC_DECL(avc), struct fid **fidpp)
 
     AFS_STATCNT(afs_fid);
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return EIO;
 
     if (afs_NFSRootOnly && (avc == afs_globalVp))

diff --git a/src/afs/afs.h b/src/afs/afs.h
index a9877d5..f470947 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -35,8 +35,12 @@
 /* Upper bound on number of iovecs out uio routines will deal with. */
 #define        AFS_MAXIOVCNT       16
 
-
-extern int afs_shuttingdown;
+enum afs_shutdown_state {
+    AFS_RUNNING = 0,
+    AFS_FLUSHING_CB = 1,
+    AFS_SHUTDOWN = 2,
+};
+extern enum afs_shutdown_state afs_shuttingdown;
 
 /*
  * Macros to uniquely identify the AFS vfs struct

diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 554da85..3ab1527 100644 (file)
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -1366,7 +1366,7 @@ afs_CheckInit(void)
     return code;
 }
 
-int afs_shuttingdown = 0;
+enum afs_shutdown_state afs_shuttingdown = AFS_RUNNING;
 void
 afs_shutdown(void)
 {
@@ -1381,15 +1381,15 @@ afs_shutdown(void)
       return;
     }
 
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
 
-    /* Give up all of our callbacks if we can. This must be done before setting
-     * afs_shuttingdown, since it calls afs_InitReq, which will fail if
-     * afs_shuttingdown is set. */
+    afs_shuttingdown = AFS_FLUSHING_CB;
+
+    /* Give up all of our callbacks if we can. */
     afs_FlushVCBs(2);
 
-    afs_shuttingdown = 1;
+    afs_shuttingdown = AFS_SHUTDOWN;
 
     if (afs_cold_shutdown)
        afs_warn("afs: COLD ");
@@ -1510,7 +1510,7 @@ afs_shutdown(void)
     memset(&afs_stats_cmfullperf, 0, sizeof(struct afs_stats_CMFullPerf));
     afs_warn(" ALL allocated tables... ");
 
-    afs_shuttingdown = 0;
+    afs_shuttingdown = AFS_RUNNING;
     afs_warn("done\n");
 
     return;                    /* Just kill daemons for now */

diff --git a/src/afs/afs_osi_pag.c b/src/afs/afs_osi_pag.c
index 309dcaa..9e86802 100644 (file)
--- a/src/afs/afs_osi_pag.c
+++ b/src/afs/afs_osi_pag.c
@@ -32,7 +32,7 @@
 
 
 /* Imported variables */
-extern int afs_shuttingdown;
+extern enum afs_shutdown_state afs_shuttingdown;
 
 /* Exported variables */
 afs_uint32 pag_epoch;
@@ -455,7 +455,7 @@ afs_InitReq(struct vrequest *av, afs_ucred_t *acred)
 
     AFS_STATCNT(afs_InitReq);
     memset(av, 0, sizeof(*av));
-    if (afs_shuttingdown)
+    if (afs_shuttingdown == AFS_SHUTDOWN)
        return EIO;
 
 #ifdef AFS_LINUX26_ENV
@@ -506,7 +506,7 @@ afs_CreateReq(struct vrequest **avpp, afs_ucred_t *acred)
     int code;
     struct vrequest *treq = NULL;
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown == AFS_SHUTDOWN) {
        return EIO;
     }
     if (!avpp || !acred) {

diff --git a/src/afs/afs_pag_call.c b/src/afs/afs_pag_call.c
index e38cf93..d7a3113 100644 (file)
--- a/src/afs/afs_pag_call.c
+++ b/src/afs/afs_pag_call.c
@@ -31,7 +31,7 @@
 
 afs_int32 afs_termState = 0;
 afs_int32 afs_gcpags = AFS_GCPAGS;
-int afs_shuttingdown = 0;
+enum afs_shutdown_state afs_shuttingdown = AFS_RUNNING;
 int afs_cold_shutdown = 0;
 int afs_resourceinit_flag = 0;
 afs_int32 afs_nfs_server_addr;
@@ -158,9 +158,9 @@ afspag_Init(afs_int32 nfs_server_addr)
 void
 afspag_Shutdown(void)
 {
-    if (afs_shuttingdown)
+    if (afs_shuttingdown != AFS_RUNNING)
        return;
-    afs_shuttingdown = 1;
+    afs_shuttingdown = AFS_SHUTDOWN;
     afs_termState = AFSOP_STOP_RXCALLBACK;
     rx_WakeupServerProcs();
     while (afs_termState == AFSOP_STOP_RXCALLBACK)

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index 87ec2fc..c2b0d5a 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -261,7 +261,7 @@ afs_FlushVCache(struct vcache *avc, int *slept)
      * via which someone could try to use the vcache. It is okay to drop
      * afs_xvcache at this point (if *slept is set). */
 
-    if (!afs_shuttingdown)
+    if (afs_shuttingdown == AFS_RUNNING)
        afs_QueueVCB(avc, slept);
 
     /*

diff --git a/src/rx/SOLARIS/rx_knet.c b/src/rx/SOLARIS/rx_knet.c
index 101356b..fe85e63 100644 (file)
--- a/src/rx/SOLARIS/rx_knet.c
+++ b/src/rx/SOLARIS/rx_knet.c
@@ -758,7 +758,7 @@ osi_NetIfPoller()
     if (li)
         (void) ldi_ident_release(li);
 
-    if (afs_shuttingdown) {
+    if (afs_shuttingdown != AFS_RUNNING) {
        /* do not schedule to run again if we're shutting down */
        return;
     }