From: Rainer Toebbicke <rtb@pclella.cern.ch>
Date: Wed, 23 Jun 2010 11:08:47 +0000 (+0200)
Subject: Do not call afs_FlushVCBs with afs_xvcache held
X-Git-Tag: openafs-devel-1_5_75~18
X-Git-Url: https://git.openafs.org/?p=openafs.git;a=commitdiff_plain;h=a309e274632993c5aeec04c6e090f5ac95837a40

Do not call afs_FlushVCBs with afs_xvcache held

In afs_AllocCBR, use dynamically created afs_cbr structures
when running out of preformatted ones, rather than calling
afs_FlushVCBs under, potentially, the afs_xvcache lock
(which would be held across the RPC, difficult to drop and
re-acquire under the current hierarchy).

Modest modernization of the number preformatted afs_cbr structures,
rule-of-thumb, not pretending any research.

Change-Id: I9427427d5dab7d4639822c370bdded0418f67d9e
Reviewed-on: http://gerrit.openafs.org/2243
Reviewed-by: Derrick Brashear <shadow@dementia.org>
Tested-by: Derrick Brashear <shadow@dementia.org>
---

diff --git a/src/afs/afs.h b/src/afs/afs.h
index 0168ceb..1c8357f 100644
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -85,8 +85,8 @@ extern int afs_shuttingdown;
 #define MAXNUMSYSNAMES	32	/* max that current constants allow */
 #define	NOTOKTIMEOUT	(2*3600)	/* time after which to timeout conns sans tokens */
 #define	NOPAG		0xffffffff
-#define AFS_NCBRS	300	/* max # of call back return entries */
-#define AFS_MAXCBRSCALL	16	/* max to return in a given call */
+#define AFS_NCBRS	1024	/* max # of call back return entries */
+#define AFS_MAXCBRSCALL	32	/* max to return in a given call (must be <= AFSCBMAX) */
 #define	AFS_SALLOC_LOW_WATER	250	/* Min free blocks before allocating more */
 #define	AFS_LRALLOCSIZ 	4096	/* "Large" allocated size */
 #define	VCACHE_FREE	5
@@ -258,6 +258,7 @@ struct afs_cbr {
     struct afs_cbr *hash_next;
 
     struct AFSFid fid;
+    unsigned int dynalloc:1;
 };
 
 /* cellinfo file magic number */
diff --git a/src/afs/afs_call.c b/src/afs/afs_call.c
index 176f0b5..bcdd5ad 100644
--- a/src/afs/afs_call.c
+++ b/src/afs/afs_call.c
@@ -1224,6 +1224,8 @@ afs_shutdown(void)
 
     if (afs_shuttingdown)
 	return;
+    afs_FlushVCBs(2);       /* Reasonable effort to free dynamically allocated callback returns */
+
     afs_shuttingdown = 1;
     if (afs_cold_shutdown)
 	afs_warn("afs: COLD ");
diff --git a/src/afs/afs_vcache.c b/src/afs/afs_vcache.c
index ef41442..ca51240 100644
--- a/src/afs/afs_vcache.c
+++ b/src/afs/afs_vcache.c
@@ -297,17 +297,21 @@ afs_InactiveVCache(struct vcache *avc, afs_ucred_t *acred)
  */
 static struct afs_cbr *afs_cbrSpace = 0;
 /* if alloc limit below changes, fix me! */
-static struct afs_cbr *afs_cbrHeads[2];
+static struct afs_cbr *afs_cbrHeads[16];
 struct afs_cbr *
 afs_AllocCBR(void)
 {
     register struct afs_cbr *tsp;
     int i;
 
-    while (!afs_cbrSpace) {
-	if (afs_stats_cmperf.CallBackAlloced >= 2) {
-	    /* don't allocate more than 2 * AFS_NCBRS for now */
-	    afs_FlushVCBs(0);
+    if (!afs_cbrSpace) {
+	afs_osi_CancelWait(&AFS_WaitHandler);	/* trigger FlushVCBs asap */
+
+	if (afs_stats_cmperf.CallBackAlloced >= sizeof(afs_cbrHeads)/sizeof(afs_cbrHeads[0])) {
+	    /* don't allocate more than 16 * AFS_NCBRS for now */
+	    tsp = (struct afs_cbr *)osi_AllocSmallSpace(sizeof(*tsp));
+	    tsp->dynalloc = 1;
+	    tsp->next = NULL;
 	    afs_stats_cmperf.CallBackFlushes++;
 	} else {
 	    /* try allocating */
@@ -316,15 +320,18 @@ afs_AllocCBR(void)
 						sizeof(struct afs_cbr));
 	    for (i = 0; i < AFS_NCBRS - 1; i++) {
 		tsp[i].next = &tsp[i + 1];
+		tsp[i].dynalloc = 0;
 	    }
 	    tsp[AFS_NCBRS - 1].next = 0;
-	    afs_cbrSpace = tsp;
+	    tsp[AFS_NCBRS - 1].dynalloc = 0;
+	    afs_cbrSpace = tsp->next;
 	    afs_cbrHeads[afs_stats_cmperf.CallBackAlloced] = tsp;
 	    afs_stats_cmperf.CallBackAlloced++;
 	}
+    } else {
+	tsp = afs_cbrSpace;
+	afs_cbrSpace = tsp->next;
     }
-    tsp = afs_cbrSpace;
-    afs_cbrSpace = tsp->next;
     return tsp;
 }
 
@@ -348,8 +355,12 @@ afs_FreeCBR(register struct afs_cbr *asp)
     if (asp->hash_next)
 	asp->hash_next->hash_pprev = asp->hash_pprev;
 
-    asp->next = afs_cbrSpace;
-    afs_cbrSpace = asp;
+    if (asp->dynalloc) {
+	osi_FreeSmallSpace(asp);
+    } else {
+	asp->next = afs_cbrSpace;
+	afs_cbrSpace = asp;
+    }
     return 0;
 }