vol: Add VInit cond var and remove busywaits

In DAFS, FSYNC_sync was waiting for VInit to reach at least 2 by
looping around pthread_yield(). For a server with a large number of
volumes, it can take a while for volumes to preattach, and so we are
effectively busy-waiting for preattach to finish. This can slow
fileserver startup and peg the cpu.

So instead, add a condition variable for when VInit changes, and wait
on that. Also modify other checkers of VInit to use the cond var.

Change-Id: Icc251fdf7d525d40a9db1938a9cb4d47a74dccba
Reviewed-on: http://gerrit.openafs.org/2648
Tested-by: Andrew Deason <adeason@sinenomine.net>
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/vol/fssync-server.c b/src/vol/fssync-server.c
index c9d6b1c..2b18048 100644 (file)
--- a/src/vol/fssync-server.c
+++ b/src/vol/fssync-server.c
@@ -266,6 +266,10 @@ FSYNC_sync(void * args)
     int min_vinit = 1;
 #endif /* AFS_DEMAND_ATTACH_FS */
 
+    /* we must not be called before vol package initialization, since we use
+     * vol package mutexes and conds etc */
+    assert(VInit);
+
     SYNC_getAddr(&state->endpoint, &state->addr);
     SYNC_cleanupSock(state);
 
@@ -282,17 +286,22 @@ FSYNC_sync(void * args)
     Log("Set thread id %d for FSYNC_sync\n", tid);
 #endif /* AFS_PTHREAD_ENV */
 
+    VOL_LOCK;
+
     while (VInit < min_vinit) {
        /* Let somebody else run until all volumes have been preattached
         * (DAFS), or we have started attaching volumes (non-DAFS). This
         * doesn't mean that all volumes have been attached.
         */
 #ifdef AFS_PTHREAD_ENV
-       pthread_yield();
+       VOL_CV_WAIT(&vol_vinit_cond);
 #else /* AFS_PTHREAD_ENV */
        LWP_DispatchProcess();
 #endif /* AFS_PTHREAD_ENV */
     }
+
+    VOL_UNLOCK;
+
     state->fd = SYNC_getSock(&state->endpoint);
     code = SYNC_bindSock(state);
     assert(!code);

diff --git a/src/vol/volume.c b/src/vol/volume.c
index 6b7d95a..b9b0ab3 100644 (file)
--- a/src/vol/volume.c
+++ b/src/vol/volume.c
@@ -156,6 +156,7 @@ pthread_mutex_t vol_trans_mutex;
 pthread_cond_t vol_put_volume_cond;
 pthread_cond_t vol_sleep_cond;
 pthread_cond_t vol_init_attach_cond;
+pthread_cond_t vol_vinit_cond;
 int vol_attach_threads = 1;
 #endif /* AFS_PTHREAD_ENV */
 
@@ -544,6 +545,22 @@ VOptDefaults(ProgramType pt, VolumePackageOptions *opts)
     }
 }
 
+/**
+ * Set VInit to a certain value, and signal waiters.
+ *
+ * @param[in] value  the value to set VInit to
+ *
+ * @pre VOL_LOCK held
+ */
+static void
+VSetVInit_r(int value)
+{
+    VInit = value;
+#ifdef AFS_PTHREAD_ENV
+    assert(pthread_cond_broadcast(&vol_vinit_cond) == 0);
+#endif
+}
+
 int
 VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts)
 {
@@ -572,6 +589,7 @@ VInitVolumePackage2(ProgramType pt, VolumePackageOptions * opts)
     assert(pthread_cond_init(&vol_put_volume_cond, NULL) == 0);
     assert(pthread_cond_init(&vol_sleep_cond, NULL) == 0);
     assert(pthread_cond_init(&vol_init_attach_cond, NULL) == 0);
+    assert(pthread_cond_init(&vol_vinit_cond, NULL) == 0);
 #else /* AFS_PTHREAD_ENV */
     IOMGR_Initialize();
 #endif /* AFS_PTHREAD_ENV */
@@ -666,7 +684,7 @@ VInitAttachVolumes(ProgramType pt)
        }
     }
     VOL_LOCK;
-    VInit = 2;                 /* Initialized, and all volumes have been attached */
+    VSetVInit_r(2);                    /* Initialized, and all volumes have been attached */
     LWP_NoYieldSignal(VInitAttachVolumes);
     VOL_UNLOCK;
     return 0;
@@ -748,7 +766,7 @@ VInitAttachVolumes(ProgramType pt)
        assert(pthread_cond_destroy(&params.thread_done_cv) == 0);
     }
     VOL_LOCK;
-    VInit = 2;                 /* Initialized, and all volumes have been attached */
+    VSetVInit_r(2);                    /* Initialized, and all volumes have been attached */
     assert(pthread_cond_broadcast(&vol_init_attach_cond) == 0);
     VOL_UNLOCK;
     return 0;
@@ -872,7 +890,7 @@ VInitAttachVolumes(ProgramType pt)
     }
 
     VOL_LOCK;
-    VInit = 2;                 /* Initialized, and all volumes have been attached */
+    VSetVInit_r(2);                    /* Initialized, and all volumes have been attached */
     assert(pthread_cond_broadcast(&vol_init_attach_cond) == 0);
     VOL_UNLOCK;
 
@@ -5430,8 +5448,9 @@ VConnectFS_r(void)
           (programType != fileServer) &&
           (programType != salvager));
     rc = FSYNC_clientInit();
-    if (rc)
-       VInit = 3;
+    if (rc) {
+       VSetVInit_r(3);
+    }
     return rc;
 }
 
@@ -5457,7 +5476,7 @@ VDisconnectFS_r(void)
     assert((programType != fileServer) &&
           (programType != salvager));
     FSYNC_clientFinis();
-    VInit = 2;
+    VSetVInit_r(2);
 }
 
 /**
@@ -6094,7 +6113,7 @@ VSetDiskUsage_r(void)
         * initialization level indicates that all volumes are attached,
         * which implies that all partitions are initialized. */
 #ifdef AFS_PTHREAD_ENV
-       sleep(10);
+       VOL_CV_WAIT(&vol_vinit_cond);
 #else /* AFS_PTHREAD_ENV */
        IOMGR_Sleep(10);
 #endif /* AFS_PTHREAD_ENV */

diff --git a/src/vol/volume.h b/src/vol/volume.h
index 8eabf84..f460212 100644 (file)
--- a/src/vol/volume.h
+++ b/src/vol/volume.h
@@ -68,6 +68,7 @@ extern pthread_mutex_t vol_glock_mutex;
 extern pthread_mutex_t vol_trans_mutex;
 extern pthread_cond_t vol_put_volume_cond;
 extern pthread_cond_t vol_sleep_cond;
+extern pthread_cond_t vol_vinit_cond;
 extern ih_init_params vol_io_params;
 extern int vol_attach_threads;
 #ifdef VOL_LOCK_DEBUG