Do not call afs_FlushVCBs with afs_xvcache held

In afs_AllocCBR, use dynamically created afs_cbr structures
when running out of preformatted ones, rather than calling
afs_FlushVCBs under, potentially, the afs_xvcache lock
(which would be held across the RPC, difficult to drop and
re-acquire under the current hierarchy).

Modest modernization of the number preformatted afs_cbr structures,
rule-of-thumb, not pretending any research.

Change-Id: I9427427d5dab7d4639822c370bdded0418f67d9e
Reviewed-on: http://gerrit.openafs.org/2243
Reviewed-by: Derrick Brashear <shadow@dementia.org>
Tested-by: Derrick Brashear <shadow@dementia.org>

diff --git a/src/afs/afs.h b/src/afs/afs.h
index 0168ceb..1c8357f 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -85,8 +85,8 @@ extern int afs_shuttingdown;
 #define MAXNUMSYSNAMES 32      /* max that current constants allow */
 #define        NOTOKTIMEOUT    (2*3600)        /* time after which to timeout conns sans tokens */
 #define        NOPAG           0xffffffff
-#define AFS_NCBRS      300     /* max # of call back return entries */
-#define AFS_MAXCBRSCALL        16      /* max to return in a given call */
+#define AFS_NCBRS      1024    /* max # of call back return entries */
+#define AFS_MAXCBRSCALL        32      /* max to return in a given call (must be <= AFSCBMAX) */
 #define        AFS_SALLOC_LOW_WATER    250     /* Min free blocks before allocating more */
 #define        AFS_LRALLOCSIZ  4096    /* "Large" allocated size */
 #define        VCACHE_FREE     5
@@ -258,6 +258,7 @@ struct afs_cbr {
     struct afs_cbr *hash_next;
 
     struct AFSFid fid;
+    unsigned int dynalloc:1;
 };
 
 /* cellinfo file magic number */

diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 176f0b5..bcdd5ad 100644 (file)
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -1224,6 +1224,8 @@ afs_shutdown(void)
 
     if (afs_shuttingdown)
        return;
+    afs_FlushVCBs(2);       /* Reasonable effort to free dynamically allocated callback returns */
+
     afs_shuttingdown = 1;
     if (afs_cold_shutdown)
        afs_warn("afs: COLD ");

diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index ef41442..ca51240 100644 (file)
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -297,17 +297,21 @@ afs_InactiveVCache(struct vcache *avc, afs_ucred_t *acred)
  */
 static struct afs_cbr *afs_cbrSpace = 0;
 /* if alloc limit below changes, fix me! */
-static struct afs_cbr *afs_cbrHeads[2];
+static struct afs_cbr *afs_cbrHeads[16];
 struct afs_cbr *
 afs_AllocCBR(void)
 {
     register struct afs_cbr *tsp;
     int i;
 
-    while (!afs_cbrSpace) {
-       if (afs_stats_cmperf.CallBackAlloced >= 2) {
-           /* don't allocate more than 2 * AFS_NCBRS for now */
-           afs_FlushVCBs(0);
+    if (!afs_cbrSpace) {
+       afs_osi_CancelWait(&AFS_WaitHandler);   /* trigger FlushVCBs asap */
+
+       if (afs_stats_cmperf.CallBackAlloced >= sizeof(afs_cbrHeads)/sizeof(afs_cbrHeads[0])) {
+           /* don't allocate more than 16 * AFS_NCBRS for now */
+           tsp = (struct afs_cbr *)osi_AllocSmallSpace(sizeof(*tsp));
+           tsp->dynalloc = 1;
+           tsp->next = NULL;
            afs_stats_cmperf.CallBackFlushes++;
        } else {
            /* try allocating */
@@ -316,15 +320,18 @@ afs_AllocCBR(void)
                                                sizeof(struct afs_cbr));
            for (i = 0; i < AFS_NCBRS - 1; i++) {
                tsp[i].next = &tsp[i + 1];
+               tsp[i].dynalloc = 0;
            }
            tsp[AFS_NCBRS - 1].next = 0;
-           afs_cbrSpace = tsp;
+           tsp[AFS_NCBRS - 1].dynalloc = 0;
+           afs_cbrSpace = tsp->next;
            afs_cbrHeads[afs_stats_cmperf.CallBackAlloced] = tsp;
            afs_stats_cmperf.CallBackAlloced++;
        }
+    } else {
+       tsp = afs_cbrSpace;
+       afs_cbrSpace = tsp->next;
     }
-    tsp = afs_cbrSpace;
-    afs_cbrSpace = tsp->next;
     return tsp;
 }
 
@@ -348,8 +355,12 @@ afs_FreeCBR(register struct afs_cbr *asp)
     if (asp->hash_next)
        asp->hash_next->hash_pprev = asp->hash_pprev;
 
-    asp->next = afs_cbrSpace;
-    afs_cbrSpace = asp;
+    if (asp->dynalloc) {
+       osi_FreeSmallSpace(asp);
+    } else {
+       asp->next = afs_cbrSpace;
+       afs_cbrSpace = asp;
+    }
     return 0;
 }