Windows: Performing async work after cache invalidation
authorPeter Scott <pscott@kerneldrivers.com>
Wed, 11 Jan 2012 13:49:23 +0000 (06:49 -0700)
committerJeffrey Altman <jaltman@secure-endpoints.com>
Wed, 18 Jan 2012 22:53:28 +0000 (14:53 -0800)
The code now queues a work item to perform additional work on extent
processing after a cache invalidation has occurred. This additional work
involves walking the current list of extents and purging/flushing regions of
the system cache based upon the current state of the extent.
Additional changes to filter which invlidation events result in a queued
worker to perform asynchronous work.

Change-Id: I72e4e0bac2caf69e41a095ce8fc4c2e083702b5c
Reviewed-on: http://gerrit.openafs.org/6528
Tested-by: BuildBot <buildbot@rampaginggeek.com>
Reviewed-by: Jeffrey Altman <jaltman@secure-endpoints.com>
Tested-by: Jeffrey Altman <jaltman@secure-endpoints.com>

src/WINNT/afsrdr/kernel/lib/AFSExtentsSupport.cpp
src/WINNT/afsrdr/kernel/lib/AFSGeneric.cpp
src/WINNT/afsrdr/kernel/lib/AFSWorker.cpp
src/WINNT/afsrdr/kernel/lib/Include/AFSCommon.h
src/WINNT/afsrdr/kernel/lib/Include/AFSDefines.h
src/WINNT/afsrdr/kernel/lib/Include/AFSStructs.h

index 14251e3..f64381c 100644 (file)
@@ -39,7 +39,6 @@
 
 #define AFS_MAX_FCBS_TO_DROP 10
 
-static AFSExtent *ExtentFor( PLIST_ENTRY le, ULONG SkipList );
 static AFSExtent *NextExtent( AFSExtent *Extent, ULONG SkipList );
 static ULONG ExtentsMasks[AFS_NUM_EXTENT_LISTS] = AFS_EXTENTS_MASKS;
 static VOID VerifyExtentsLists(AFSFcb *Fcb);
@@ -3169,6 +3168,264 @@ try_exit:
     return ntStatus;
 }
 
+NTSTATUS
+AFSReleaseCleanExtents( IN AFSFcb *Fcb,
+                        IN GUID *AuthGroup)
+{
+    AFSNonPagedFcb      *pNPFcb = Fcb->NPFcb;
+    AFSExtent           *pExtent;
+    LIST_ENTRY          *le;
+    AFSReleaseExtentsCB *pRelease = NULL;
+    ULONG                count = 0;
+    ULONG                initialDirtyCount = 0;
+    BOOLEAN              bExtentsLocked = FALSE;
+    ULONG                total = 0;
+    ULONG                sz = 0;
+    NTSTATUS             ntStatus = STATUS_SUCCESS;
+    LARGE_INTEGER        liLastFlush;
+    ULONG                ulRemainingExtentLength = 0;
+    AFSDeviceExt        *pControlDevExt = (AFSDeviceExt *)AFSControlDeviceObject->DeviceExtension;
+    GUID                *pAuthGroup = AuthGroup;
+    GUID                 stAuthGroup;
+
+    ASSERT( Fcb->Header.NodeTypeCode == AFS_FILE_FCB);
+
+    //
+    // Save, then reset the flush time
+    //
+
+    liLastFlush = Fcb->Specific.File.LastServerFlush;
+
+    KeQueryTickCount( &Fcb->Specific.File.LastServerFlush);
+
+    __Enter
+    {
+
+        if( pAuthGroup == NULL ||
+            RtlCompareMemory( pAuthGroup,
+                              &Fcb->NPFcb->Specific.File.ExtentsRequestAuthGroup,
+                              sizeof( GUID)) == sizeof( GUID))
+        {
+
+            RtlZeroMemory( &stAuthGroup,
+                           sizeof( GUID));
+
+            ntStatus = AFSRetrieveValidAuthGroup( Fcb,
+                                                  NULL,
+                                                  TRUE,
+                                                  &stAuthGroup);
+
+            if( !NT_SUCCESS( ntStatus))
+            {
+                try_return( ntStatus);
+            }
+
+            pAuthGroup = &stAuthGroup;
+        }
+
+        //
+        // Look for a start in the list to flush entries
+        //
+
+        total = count;
+
+        sz = sizeof( AFSReleaseExtentsCB ) + (AFS_MAXIMUM_EXTENT_RELEASE_COUNT * sizeof ( AFSFileExtentCB ));
+
+        pRelease = (AFSReleaseExtentsCB*) AFSExAllocatePoolWithTag( NonPagedPool,
+                                                                    sz,
+                                                                    AFS_EXTENT_RELEASE_TAG);
+        if( NULL == pRelease)
+        {
+
+            try_return ( ntStatus = STATUS_INSUFFICIENT_RESOURCES );
+        }
+
+        while( Fcb->Specific.File.ExtentLength > (LONG)ulRemainingExtentLength)
+        {
+
+            AFSLockForExtentsTrim( Fcb);
+
+            bExtentsLocked = TRUE;
+
+            pRelease->Flags = AFS_EXTENT_FLAG_RELEASE;
+
+            //
+            // Update the metadata for this call
+            //
+
+            pRelease->AllocationSize = Fcb->ObjectInformation->EndOfFile;
+            pRelease->CreateTime = Fcb->ObjectInformation->CreationTime;
+            pRelease->ChangeTime = Fcb->ObjectInformation->ChangeTime;
+            pRelease->LastAccessTime = Fcb->ObjectInformation->LastAccessTime;
+            pRelease->LastWriteTime = Fcb->ObjectInformation->LastWriteTime;
+
+            count = 0;
+
+            le = Fcb->Specific.File.ExtentsLists[AFS_EXTENTS_LIST].Flink;
+
+            while( count < AFS_MAXIMUM_EXTENT_RELEASE_COUNT &&
+                   le != &Fcb->Specific.File.ExtentsLists[AFS_EXTENTS_LIST])
+            {
+
+                pExtent = ExtentFor( le, AFS_EXTENTS_LIST);
+
+                le = le->Flink;
+
+                if( pExtent->ActiveCount > 0 ||
+                    BooleanFlagOn( pExtent->Flags, AFS_EXTENT_DIRTY))
+                {
+                    continue;
+                }
+
+                pRelease->FileExtents[count].Flags = AFS_EXTENT_FLAG_RELEASE;
+
+                AFSDbgLogMsg( AFS_SUBSYSTEM_EXTENT_PROCESSING,
+                              AFS_TRACE_LEVEL_VERBOSE,
+                              "AFSReleaseCleanExtents Releasing extent %p fid %08lX-%08lX-%08lX-%08lX Offset %I64X Len %08lX\n",
+                              pExtent,
+                              Fcb->ObjectInformation->FileId.Cell,
+                              Fcb->ObjectInformation->FileId.Volume,
+                              Fcb->ObjectInformation->FileId.Vnode,
+                              Fcb->ObjectInformation->FileId.Unique,
+                              pExtent->FileOffset.QuadPart,
+                              pExtent->Size);
+
+                pRelease->FileExtents[count].Length = pExtent->Size;
+                pRelease->FileExtents[count].DirtyLength = pExtent->Size;
+                pRelease->FileExtents[count].DirtyOffset = 0;
+                pRelease->FileExtents[count].CacheOffset = pExtent->CacheOffset;
+                pRelease->FileExtents[count].FileOffset = pExtent->FileOffset;
+
+#if GEN_MD5
+                RtlCopyMemory( pRelease->FileExtents[count].MD5,
+                               pExtent->MD5,
+                               sizeof(pExtent->MD5));
+
+                pRelease->FileExtents[count].Flags |= AFS_EXTENT_FLAG_MD5_SET;
+#endif
+
+                //
+                // Need to pull this extent from the main list as well
+                //
+
+                for (ULONG i = 0; i < AFS_NUM_EXTENT_LISTS; i ++)
+                {
+                    if (NULL != pExtent->Lists[i].Flink && !IsListEmpty(&pExtent->Lists[i]))
+                    {
+                        RemoveEntryList( &pExtent->Lists[i] );
+                    }
+                }
+
+                InterlockedExchangeAdd( &pControlDevExt->Specific.Control.ExtentsHeldLength, -((LONG)(pExtent->Size/1024)));
+
+                InterlockedExchangeAdd( &Fcb->Specific.File.ExtentLength, -((LONG)(pExtent->Size/1024)));
+
+                AFSExFreePool( pExtent);
+
+                InterlockedDecrement( &Fcb->Specific.File.ExtentCount);
+
+                if( InterlockedDecrement( &pControlDevExt->Specific.Control.ExtentCount) == 0)
+                {
+
+                    KeSetEvent( &pControlDevExt->Specific.Control.ExtentsHeldEvent,
+                                0,
+                                FALSE);
+                }
+
+                count ++;
+            }
+
+            //
+            // If we are done then get out
+            //
+
+            if( count == 0)
+            {
+
+                AFSDbgLogMsg( AFS_SUBSYSTEM_EXTENT_PROCESSING,
+                              AFS_TRACE_LEVEL_VERBOSE,
+                              "AFSReleaseCleanExtents No more dirty extents found\n");
+
+                break;
+            }
+
+            //
+            // Fire off the request synchronously
+            //
+
+            sz = sizeof( AFSReleaseExtentsCB ) + (count * sizeof ( AFSFileExtentCB ));
+
+            pRelease->ExtentCount = count;
+
+            //
+            // Drop the extents lock for the duration of the call to
+            // the network.  We have pinned the extents so, even
+            // though we might get extents added during this period,
+            // but none will be removed.  Hence we can carry on from
+            // le.
+            //
+
+            AFSDbgLogMsg( AFS_SUBSYSTEM_LOCK_PROCESSING,
+                          AFS_TRACE_LEVEL_VERBOSE,
+                          "AFSReleaseCleanExtents Releasing Fcb extents lock %08lX thread %08lX\n",
+                          &pNPFcb->Specific.File.ExtentsResource,
+                          PsGetCurrentThread());
+
+            AFSReleaseResource( &pNPFcb->Specific.File.ExtentsResource);
+            bExtentsLocked = FALSE;
+
+            ntStatus = AFSProcessRequest( AFS_REQUEST_TYPE_RELEASE_FILE_EXTENTS,
+                                          AFS_REQUEST_FLAG_SYNCHRONOUS,
+                                          pAuthGroup,
+                                          NULL,
+                                          &Fcb->ObjectInformation->FileId,
+                                          pRelease,
+                                          sz,
+                                          NULL,
+                                          NULL);
+
+            if( !NT_SUCCESS(ntStatus))
+            {
+
+                //
+                // Regardless of whether or not the AFSProcessRequest() succeeded, the extents
+                // were released (if AFS_EXTENT_FLAG_RELEASE was set).  Log the error so it is known.
+                //
+
+                AFSDbgLogMsg( AFS_SUBSYSTEM_EXTENT_PROCESSING,
+                              AFS_TRACE_LEVEL_ERROR,
+                              "AFSReleaseCleanExtents AFS_REQUEST_TYPE_RELEASE_FILE_EXTENTS failed fid %08lX-%08lX-%08lX-%08lX Status %08lX\n",
+                              Fcb->ObjectInformation->FileId.Cell,
+                              Fcb->ObjectInformation->FileId.Volume,
+                              Fcb->ObjectInformation->FileId.Vnode,
+                              Fcb->ObjectInformation->FileId.Unique,
+                              ntStatus);
+            }
+        }
+
+try_exit:
+
+        if (bExtentsLocked)
+        {
+
+            AFSDbgLogMsg( AFS_SUBSYSTEM_LOCK_PROCESSING,
+                          AFS_TRACE_LEVEL_VERBOSE,
+                          "AFSReleaseCleanExtents Releasing Fcb extents lock %08lX thread %08lX\n",
+                          &pNPFcb->Specific.File.ExtentsResource,
+                          PsGetCurrentThread());
+
+            AFSReleaseResource( &pNPFcb->Specific.File.ExtentsResource );
+        }
+
+        if (pRelease)
+        {
+            AFSExFreePool( pRelease);
+        }
+    }
+
+    return ntStatus;
+}
+
 VOID
 AFSMarkDirty( IN AFSFcb *Fcb,
               IN AFSExtent *StartExtent,
@@ -3345,7 +3602,8 @@ AFSMarkDirty( IN AFSFcb *Fcb,
 // Helper functions
 //
 
-static AFSExtent *ExtentFor(PLIST_ENTRY le, ULONG SkipList)
+AFSExtent *
+ExtentFor(PLIST_ENTRY le, ULONG SkipList)
 {
     return CONTAINING_RECORD( le, AFSExtent, Lists[SkipList] );
 }
index c528232..ca071ed 100644 (file)
@@ -1819,19 +1819,10 @@ AFSInvalidateCache( IN AFSInvalidateCacheCB *InvalidateCB)
                                                 ulFilter,
                                                 FILE_ACTION_REMOVED);
 
-                if( pObjectInfo->FileType == AFS_FILE_TYPE_FILE &&
-                    pObjectInfo->Fcb != NULL)
+                if( NT_SUCCESS( AFSQueueInvalidateObject( pObjectInfo,
+                                                          InvalidateCB->Reason)))
                 {
-
-
-                    //
-                    // Clear out the extents
-                    // And get rid of them (note this involves waiting
-                    // for any writes or reads to the cache to complete)
-                    //
-
-                    (VOID) AFSTearDownFcbExtents( pObjectInfo->Fcb,
-                                                  NULL);
+                    pObjectInfo = NULL; // We'll dec the count in the worker item
                 }
 
                 break;
@@ -1969,6 +1960,13 @@ AFSInvalidateCache( IN AFSInvalidateCacheCB *InvalidateCB)
 
                 SetFlag( pObjectInfo->Flags, AFS_OBJECT_FLAGS_VERIFY);
 
+                if( InvalidateCB->Reason == AFS_INVALIDATE_DATA_VERSION &&
+                    NT_SUCCESS( AFSQueueInvalidateObject( pObjectInfo,
+                                                          InvalidateCB->Reason)))
+                {
+                    pObjectInfo = NULL; // We'll dec the count in the worker item
+                }
+
                 break;
             }
         }
@@ -8524,3 +8522,112 @@ try_exit:
 
     return ntStatus;
 }
+
+NTSTATUS
+AFSPerformObjectInvalidate( IN AFSObjectInfoCB *ObjectInfo,
+                            IN ULONG InvalidateReason)
+{
+
+    NTSTATUS            ntStatus = STATUS_SUCCESS;
+    IO_STATUS_BLOCK     stIoStatus;
+    LIST_ENTRY         *le;
+    AFSExtent          *pEntry;
+    ULONG               ulProcessCount = 0;
+    ULONG               ulCount = 0;
+
+    __Enter
+    {
+
+        switch( InvalidateReason)
+        {
+
+            case AFS_INVALIDATE_DELETED:
+            {
+
+                if( ObjectInfo->FileType == AFS_FILE_TYPE_FILE &&
+                    ObjectInfo->Fcb != NULL)
+                {
+
+
+                    //
+                    // Clear out the extents
+                    // And get rid of them (note this involves waiting
+                    // for any writes or reads to the cache to complete)
+                    //
+
+                    (VOID) AFSTearDownFcbExtents( ObjectInfo->Fcb,
+                                                  NULL);
+                }
+
+                break;
+            }
+
+            case AFS_INVALIDATE_DATA_VERSION:
+            {
+
+                if( ObjectInfo->FileType == AFS_FILE_TYPE_FILE &&
+                    ObjectInfo->Fcb != NULL)
+                {
+
+                    AFSAcquireExcl( &ObjectInfo->Fcb->NPFcb->Resource,
+                                    TRUE);
+
+                    AFSLockForExtentsTrim( ObjectInfo->Fcb);
+
+                    __try
+                    {
+
+                        le = ObjectInfo->Fcb->Specific.File.ExtentsLists[AFS_EXTENTS_LIST].Flink;
+
+                        ulProcessCount = 0;
+
+                        ulCount = (ULONG)ObjectInfo->Fcb->Specific.File.ExtentCount;
+
+                        while( ulProcessCount < ulCount)
+                        {
+                            pEntry = ExtentFor( le, AFS_EXTENTS_LIST );
+
+                            if( !BooleanFlagOn( pEntry->Flags, AFS_EXTENT_DIRTY))
+                            {
+                                CcPurgeCacheSection( &ObjectInfo->Fcb->NPFcb->SectionObjectPointers,
+                                                     &pEntry->FileOffset,
+                                                     pEntry->Size,
+                                                     FALSE);
+                            }
+
+                            ulProcessCount++;
+                            le = le->Flink;
+                        }
+                    }
+                    __except( EXCEPTION_EXECUTE_HANDLER)
+                    {
+
+                        ntStatus = GetExceptionCode();
+                    }
+
+                    AFSReleaseResource( &ObjectInfo->Fcb->NPFcb->Specific.File.ExtentsResource );
+
+                    AFSReleaseResource( &ObjectInfo->Fcb->NPFcb->Resource);
+
+                    AFSReleaseCleanExtents( ObjectInfo->Fcb,
+                                            NULL);
+                }
+
+                break;
+            }
+
+            default:
+            {
+
+                break;
+            }
+        }
+
+        if( ObjectInfo != NULL)
+        {
+            InterlockedDecrement( &ObjectInfo->ObjectReferenceCount);
+        }
+    }
+
+    return ntStatus;
+}
index 1074d2b..7507a57 100644 (file)
@@ -737,6 +737,17 @@ AFSWorkerThread( IN PVOID Context)
                         break;
                     }
 
+                    case AFS_WORK_INVALIDATE_OBJECT:
+                    {
+
+                        AFSPerformObjectInvalidate( pWorkItem->Specific.Invalidate.ObjectInfo,
+                                                    pWorkItem->Specific.Invalidate.InvalidateReason);
+
+                        freeWorkItem = TRUE;
+
+                        break;
+                    }
+
                     case AFS_WORK_START_IOS:
                     {
 
@@ -2508,3 +2519,77 @@ try_exit:
 
     return ntStatus;
 }
+
+NTSTATUS
+AFSQueueInvalidateObject( IN AFSObjectInfoCB *ObjectInfo,
+                          IN ULONG InvalidateReason)
+{
+
+    NTSTATUS ntStatus = STATUS_SUCCESS;
+    AFSWorkItem *pWorkItem = NULL;
+
+    __try
+    {
+
+        pWorkItem = (AFSWorkItem *) AFSLibExAllocatePoolWithTag( NonPagedPool,
+                                                                 sizeof(AFSWorkItem),
+                                                                 AFS_WORK_ITEM_TAG);
+        if (NULL == pWorkItem)
+        {
+
+            AFSDbgLogMsg( AFS_SUBSYSTEM_IO_PROCESSING,
+                          AFS_TRACE_LEVEL_ERROR,
+                          "AFSQueueInvalidateObject Failed to allocate work item\n");
+
+            try_return( ntStatus = STATUS_INSUFFICIENT_RESOURCES );
+        }
+
+        RtlZeroMemory( pWorkItem,
+                       sizeof(AFSWorkItem));
+
+        pWorkItem->Size = sizeof( AFSWorkItem);
+
+        pWorkItem->RequestType = AFS_WORK_INVALIDATE_OBJECT;
+
+        pWorkItem->Specific.Invalidate.ObjectInfo = ObjectInfo;
+
+        pWorkItem->Specific.Invalidate.InvalidateReason = InvalidateReason;
+
+        AFSDbgLogMsg( AFS_SUBSYSTEM_WORKER_PROCESSING,
+                      AFS_TRACE_LEVEL_VERBOSE,
+                      "AFSQueueInvalidateObject Workitem %08lX\n",
+                      pWorkItem);
+
+        ntStatus = AFSQueueWorkerRequest( pWorkItem);
+
+try_exit:
+
+        AFSDbgLogMsg( AFS_SUBSYSTEM_WORKER_PROCESSING,
+                      AFS_TRACE_LEVEL_VERBOSE,
+                      "AFSQueueInvalidateObject Request complete Status %08lX\n",
+                      ntStatus);
+
+        if( !NT_SUCCESS( ntStatus))
+        {
+
+            if( pWorkItem != NULL)
+            {
+                ExFreePoolWithTag( pWorkItem, AFS_WORK_ITEM_TAG);
+            }
+
+            AFSDbgLogMsg( AFS_SUBSYSTEM_FILE_PROCESSING,
+                          AFS_TRACE_LEVEL_ERROR,
+                          "AFSQueueInvalidateObject Failed to queue request Status %08lX\n",
+                          ntStatus);
+        }
+    }
+    __except( AFSExceptionFilter( GetExceptionCode(), GetExceptionInformation()) )
+    {
+
+        AFSDbgLogMsg( 0,
+                      0,
+                      "EXCEPTION - AFSQueueInvalidateObject\n");
+    }
+
+    return ntStatus;
+}
index adfd265..ba92d97 100644 (file)
@@ -391,6 +391,10 @@ NTSTATUS
 AFSReleaseExtentsWithFlush( IN AFSFcb *Fcb,
                             IN GUID *AuthGroup);
 
+NTSTATUS
+AFSReleaseCleanExtents( IN AFSFcb *Fcb,
+                        IN GUID *AuthGroup);
+
 VOID
 AFSMarkDirty( IN AFSFcb *pFcb,
               IN AFSExtent *StartExtent,
@@ -422,6 +426,9 @@ void
 AFSRemoveEntryDirtyList( IN AFSFcb *Fcb,
                          IN AFSExtent *Extent);
 
+AFSExtent *
+ExtentFor( PLIST_ENTRY le, ULONG SkipList );
+
 #if GEN_MD5
 void
 AFSSetupMD5Hash( IN AFSFcb *Fcb,
@@ -1373,6 +1380,10 @@ AFSRetrieveValidAuthGroup( IN AFSFcb *Fcb,
                            IN BOOLEAN WriteAccess,
                            OUT GUID *AuthGroup);
 
+NTSTATUS
+AFSPerformObjectInvalidate( IN AFSObjectInfoCB *ObjectInfo,
+                            IN ULONG InvalidateReason);
+
 //
 // AFSWorker.cpp Prototypes
 //
@@ -1466,6 +1477,10 @@ AFSQueueStartIos( IN PFILE_OBJECT CacheFileObject,
                   IN ULONG RunCount,
                   IN AFSGatherIo *GatherIo);
 
+NTSTATUS
+AFSQueueInvalidateObject( IN AFSObjectInfoCB *ObjectInfo,
+                          IN ULONG InvalidateReason);
+
 //
 // AFSMD5Support.cpp Prototypes
 //
index 7f1c39e..d7c102e 100644 (file)
@@ -116,7 +116,7 @@ NTSTATUS
 #define AFS_WORK_ASYNCH_WRITE                   0x0004
 #define AFS_WORK_UNUSED_5                       0x0005
 #define AFS_WORK_ENUMERATE_GLOBAL_ROOT          0x0006
-#define AFS_WORK_UNUSED_7                       0x0007
+#define AFS_WORK_INVALIDATE_OBJECT              0x0007
 #define AFS_WORK_START_IOS                      0x0008
 
 //
index a2c14b5..c410892 100644 (file)
@@ -636,6 +636,15 @@ typedef struct _AFS_WORK_ITEM
 
         struct
         {
+
+            AFSObjectInfoCB *ObjectInfo;
+
+            ULONG            InvalidateReason;
+
+        } Invalidate;
+
+        struct
+        {
             char     Context[ 1];
 
         } Other;