SOLARIS: Granular multiPage detection

Currently, a struct vcache has a multiPage counter, indicating how
many afs_getpage requests are in-flight for that vcache that involve
retrieving multiple pages. Any dcache associated with such vcaches are
then avoided when choosing dcache entries to evict from the cache,
since we may deadlock when trying to evict a dcache entry from one of
the earlier afs_GetOnePage calls in a particular afs_getpage request.

This behavior can cause the client to become unusable if the cache
becomes full, and the only items in the cache are dcache entries in a
file that has an in-flight multi-page afs_getpage request. Since, in
that case, we cannot kick out any entries from the cache, and so we
wait forever to wait for the cache utilization to go down.

To prevent this from occurring, record exactly which ranges in the
file have in-flight multi-page afs_getpage requests, and just avoid
dcache entries in those ranges. This way afs_GetDownD can evict dcache
entries in the same file, but still avoid entries that would cause a
deadlock.

Also add some comments explaining this situation a bit more.

Change-Id: Idb305c8b7511065301739542772d16d4fe8cd574
Reviewed-on: http://gerrit.openafs.org/4896
Reviewed-by: Derrick Brashear <shadow@dementia.org>
Tested-by: BuildBot <buildbot@rampaginggeek.com>

diff --git a/src/afs/SOLARIS/osi_vcache.c b/src/afs/SOLARIS/osi_vcache.c
index e5a98be..4c40a74 100644 (file)
--- a/src/afs/SOLARIS/osi_vcache.c
+++ b/src/afs/SOLARIS/osi_vcache.c
@@ -39,6 +39,8 @@ void
 osi_PrePopulateVCache(struct vcache *avc) {
     memset(avc, 0, sizeof(struct vcache));
 
+    QInit(&avc->multiPage);
+
     AFS_RWLOCK_INIT(&avc->vlock, "vcache vlock");
 
     rw_init(&avc->rwlock, "vcache rwlock", RW_DEFAULT, NULL);

diff --git a/src/afs/SOLARIS/osi_vm.c b/src/afs/SOLARIS/osi_vm.c
index ece7471..10fbb33 100644 (file)
--- a/src/afs/SOLARIS/osi_vm.c
+++ b/src/afs/SOLARIS/osi_vm.c
@@ -54,6 +54,34 @@ osi_VM_GetDownD(struct vcache *avc, struct dcache *adc)
     return code;
 }
 
+/* Does this dcache conflict with a multiPage request for this vcache?
+ *
+ * This function only exists for Solaris. This is used by afs_GetDownD to
+ * calculate if trying to evict the given dcache may deadlock with an
+ * in-progress afs_getpage call that is trying to get more than one page at
+ * once. See afs_getpage for details. We return 0 if we do NOT conflict,
+ * nonzero otherwise. If we return nonzero, we should NOT try to evict the
+ * given dcache entry from the cache.
+ *
+ * Locking: tvc->vlock is write-locked on entry (and GLOCK is held)
+ */
+int
+osi_VM_MultiPageConflict(struct vcache *avc, struct dcache *adc)
+{
+    struct multiPage_range *range;
+    for (range = (struct multiPage_range *)avc->multiPage.next;
+         range != &avc->multiPage;
+         range = (struct multiPage_range *)QNext(&range->q)) {
+
+       if (adc->f.chunk >= AFS_CHUNK(range->off) &&
+           adc->f.chunk <= AFS_CHUNK(range->off + range->len - 1)) {
+           return 1;
+       }
+    }
+
+    return 0;
+}
+
 /* Try to discard pages, in order to recycle a vcache entry.
  *
  * We also make some sanity checks:  ref count, open count, held locks.

diff --git a/src/afs/SOLARIS/osi_vnodeops.c b/src/afs/SOLARIS/osi_vnodeops.c
index 514294e..b0076ec 100644 (file)
--- a/src/afs/SOLARIS/osi_vnodeops.c
+++ b/src/afs/SOLARIS/osi_vnodeops.c
@@ -158,14 +158,39 @@ afs_getpage(struct vnode *vp, offset_t off, u_int len, u_int *protp,
        code =
            afs_GetOnePage(vp, off, len, protp, pl, plsz, seg, addr, rw, acred);
     else {
+       struct multiPage_range range;
        struct vcache *vcp = VTOAFS(vp);
+
+       /* We've been asked to get more than one page. We must return all
+        * requested pages at once, all of them locked, which means all of
+        * these dcache entries cannot be kicked out of the cache before we
+        * return (since their pages cannot be invalidated).
+        *
+        * afs_GetOnePage will be called multiple times by pvn_getpages in
+        * order to get all of the requested pages. One of the later
+        * afs_GetOnePage calls may need to evict some cache entries in order
+        * to perform its read. If we try to kick out one of the entries an
+        * earlier afs_GetOnePage call used, we will deadlock since we have
+        * the page locked. So, to tell afs_GetDownD that it should skip over
+        * any entries we've read in due to this afs_getpage call, record the
+        * offset and length in avc->multiPage.
+        *
+        * Ideally we would just set something in each dcache as we get it,
+        * but that is rather difficult, since pvn_getpages doesn't let us
+        * retain any information between calls to afs_GetOnePage. So instead
+        * just record the offset and length, and let afs_GetDownD calculate
+        * which dcache entries should be skipped. */
+
+       range.off = off;
+       range.len = len;
+
        ObtainWriteLock(&vcp->vlock, 548);
-       vcp->multiPage++;
+       QAdd(&vcp->multiPage, &range.q);
        ReleaseWriteLock(&vcp->vlock);
        code =
            pvn_getpages(afs_GetOnePage, vp, off, len, protp, pl, plsz, seg, addr, rw, acred);
        ObtainWriteLock(&vcp->vlock, 549);
-       vcp->multiPage--;
+       QRemove(&range.q);
        ReleaseWriteLock(&vcp->vlock);
     }
     AFS_GUNLOCK();

diff --git a/src/afs/afs.h b/src/afs/afs.h
index ed2e8a2..914c8f1 100644 (file)
--- a/src/afs/afs.h
+++ b/src/afs/afs.h
@@ -799,6 +799,21 @@ struct fvcache {
     struct afs_vnuniq oldParent;
 };
 
+#ifdef AFS_SUN5_ENV
+/*
+ * This is for the multiPage field in struct vcache. Each one of these
+ * represents an outstanding getpage request that is larger than a single page.
+ * Recording these is necessary to prevent afs_GetOnePage from trying to evict
+ * a dcache entry that an earlier afs_GetOnePage call got in the same getpage
+ * request. See osi_VM_MultiPageConflict and afs_getpage.
+ */
+struct multiPage_range {
+    struct afs_q q;
+    offset_t off;    /**< offset of getpage request */
+    u_int len;       /**< length of getpage request */
+};
+#endif
+
 /* INVARIANTs: (vlruq.next != NULL) == (vlruq.prev != NULL)
  *             nextfree => !vlruq.next && ! vlruq.prev
  * !(avc->nextfree) && !avc->vlruq.next => (FreeVCList == avc->nextfree)
@@ -919,7 +934,7 @@ struct vcache {
     afs_ucred_t *uncred;
     int asynchrony;            /* num kbytes to store behind */
 #ifdef AFS_SUN5_ENV
-    short multiPage;           /* count of multi-page getpages in progress */
+    struct afs_q multiPage;    /* list of multiPage_range structs */
 #endif
     int protocol;              /* RX_FILESERVER, RX_OSD, ... defined in afsint.xg */
 #if !defined(UKERNEL)

diff --git a/src/afs/afs_dcache.c b/src/afs/afs_dcache.c
index 0701224..3d6f2d1 100644 (file)
--- a/src/afs/afs_dcache.c
+++ b/src/afs/afs_dcache.c
@@ -589,7 +589,9 @@ afs_GetDownD(int anumber, int *aneedSpace, afs_int32 buckethint)
      * we don't reclaim active entries, or other than target bucket.
      * Set to 1, we reclaim even active ones in target bucket.
      * Set to 2, we reclaim any inactive one.
-     * Set to 3, we reclaim even active ones.
+     * Set to 3, we reclaim even active ones. On Solaris, we also reclaim
+     * entries whose corresponding vcache has a nonempty multiPage list, when
+     * possible.
      */
     if (splitdcache) {
        phase = 0;
@@ -724,9 +726,11 @@ afs_GetDownD(int anumber, int *aneedSpace, afs_int32 buckethint)
 
                        ReleaseWriteLock(&afs_xdcache);
                        ObtainWriteLock(&tvc->vlock, 543);
-                       if (tvc->multiPage) {
-                           skip = 1;
-                           goto endmultipage;
+                       if (!QEmpty(&tvc->multiPage)) {
+                           if (phase < 3 || osi_VM_MultiPageConflict(tvc, tdc)) {
+                               skip = 1;
+                               goto endmultipage;
+                           }
                        }
                        /* block locking pages */
                        tvc->vstates |= VPageCleaning;

diff --git a/src/afs/afs_prototypes.h b/src/afs/afs_prototypes.h
index 1f1e15f..ee7018d 100644 (file)
--- a/src/afs/afs_prototypes.h
+++ b/src/afs/afs_prototypes.h
@@ -764,6 +764,7 @@ extern int osi_VM_Setup(struct vcache *avc, int force);
 
 #ifdef AFS_SUN5_ENV
 extern int osi_VM_GetDownD(struct vcache *avc, struct dcache *adc);
+extern int osi_VM_MultiPageConflict(struct vcache *avc, struct dcache *adc);
 extern void osi_VM_PreTruncate(struct vcache *avc, int alen,
                               afs_ucred_t *acred);
 #endif