[NTOS]: Add MiRemoveZeroPageSafe helper function, when a zero page is required, but the inline zeroing of MiRemoveZeroPage is not. This function will only try grabbing a zero page if one exists, otherwise a free page will be grabbed and zeroed with custom code of the caller's choosing.

[NTOS]: Add concept of process color and system color. Compute correct color to use whenever requesting a page.
[NTOS]: Uncondtionally enable the color code when inserting/removing pages.
For now, when requesting a page, colors are still ignored, and the global PFN lists are scanned instead. If there are no regressions, we are one patch away from that.

svn path=/trunk/; revision=48927

Author: Sir Richard

reactos/ntoskrnl/mm/ARM3/miarm.h

@@ -218,6 +218,13 @@ extern const ULONG MmProtectToPteMask[32];
 //
 #define MM_NOIRQL (KIRQL)0xFFFFFFFF
 
+//
+// Returns the color of a page
+//
+#define MI_GET_PAGE_COLOR(x)                ((x) & MmSecondaryColorMask)
+#define MI_GET_NEXT_COLOR(x)                (MI_GET_PAGE_COLOR(++MmSystemPageColor))
+#define MI_GET_NEXT_PROCESS_COLOR(x)        (MI_GET_PAGE_COLOR(++(x)->NextPageColor))
+
 //
 // FIXFIX: These should go in ex.h after the pool merge
 //
@@ -455,9 +462,8 @@ extern PMMPTE MmSharedUserDataPte;
 extern LIST_ENTRY MmProcessList;
 extern BOOLEAN MmZeroingPageThreadActive;
 extern KEVENT MmZeroingPageEvent;
-
-#define MI_PFN_TO_PFNENTRY(x)     (&MmPfnDatabase[1][x])
-#define MI_PFNENTRY_TO_PFN(x)     (x - MmPfnDatabase[1])
+extern ULONG MmSystemPageColor;
+extern ULONG MmProcessColorSeed;
 
 //
 // Figures out the hardware bits for a PTE
@@ -1093,4 +1099,19 @@ MiGetNextNode(
     IN PMMADDRESS_NODE Node
 );
 
+//
+// MiRemoveZeroPage will use inline code to zero out the page manually if only
+// free pages are available. In some scenarios, we don't/can't run that piece of
+// code and would rather only have a real zero page. If we can't have a zero page,
+// then we'd like to have our own code to grab a free page and zero it out, by
+// using MiRemoveAnyPage. This macro implements this.
+//
+PFN_NUMBER
+FORCEINLINE
+MiRemoveZeroPageSafe(IN ULONG Color)
+{
+    if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color);
+    return 0;
+}
+
 /* EOF */

reactos/ntoskrnl/mm/ARM3/pagfault.c

@@ -179,9 +179,10 @@ MiResolveDemandZeroFault(IN PVOID Address,
                          IN PEPROCESS Process,
                          IN KIRQL OldIrql)
 {
-    PFN_NUMBER PageFrameNumber;
+    PFN_NUMBER PageFrameNumber = 0;
     MMPTE TempPte;
     BOOLEAN NeedZero = FALSE;
+    ULONG Color;
     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
             Address,
             Process);
@@ -196,19 +197,39 @@ MiResolveDemandZeroFault(IN PVOID Address,
         /* No forking yet */
         ASSERT(Process->ForkInProgress == NULL);
 
+        /* Get process color */
+        Color = MI_GET_NEXT_PROCESS_COLOR(Process);
+        
         /* We'll need a zero page */
         NeedZero = TRUE;
     }
+    else
+    {
+        /* Get the next system page color */
+        Color = MI_GET_NEXT_COLOR();
+    }
 
     //
     // Lock the PFN database
     //
     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
     ASSERT(PointerPte->u.Hard.Valid == 0);
 
-    /* Get a page */
-    PageFrameNumber = MiRemoveAnyPage(0);
-    DPRINT("New pool page: %lx\n", PageFrameNumber);
+    /* Do we need a zero page? */
+    if (NeedZero)
+    {
+        /* Try to get one, if we couldn't grab a free page and zero it */
+        PageFrameNumber = MiRemoveZeroPageSafe(Color);
+        if (PageFrameNumber) NeedZero = FALSE;
+    }
+    
+    /* Did we get a page? */
+    if (!PageFrameNumber)
+    {
+        /* We either failed to find a zero page, or this is a system request */
+        PageFrameNumber = MiRemoveAnyPage(Color);
+        DPRINT("New pool page: %lx\n", PageFrameNumber);
+    }
 
     /* Initialize it */
     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
@@ -463,6 +484,7 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
     ULONG ProtectionCode;
     PMMVAD Vad;
     PFN_NUMBER PageFrameIndex;
+    ULONG Color;
     DPRINT("ARM3 FAULT AT: %p\n", Address);
 
     //
@@ -756,19 +778,25 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
 
         /* Lock the PFN database since we're going to grab a page */
         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        
+        /* Try to get a zero page */
+        Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
+        PageFrameIndex = MiRemoveZeroPageSafe(Color);
+        if (!PageFrameIndex)
+        {
+            /* Grab a page out of there. Later we should grab a colored zero page */
+            PageFrameIndex = MiRemoveAnyPage(Color);
+            ASSERT(PageFrameIndex);
 
-        /* Grab a page out of there. Later we should grab a colored zero page */
-        PageFrameIndex = MiRemoveAnyPage(0);
-        ASSERT(PageFrameIndex);
+            /* Release the lock since we need to do some zeroing */
+            KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 
-        /* Release the lock since we need to do some zeroing */
-        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+            /* Zero out the page, since it's for user-mode */
+            MiZeroPfn(PageFrameIndex);
 
-        /* Zero out the page, since it's for user-mode */
-        MiZeroPfn(PageFrameIndex);
-
-        /* Grab the lock again so we can initialize the PFN entry */
-        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+            /* Grab the lock again so we can initialize the PFN entry */
+            OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        }
 
         /* Initialize the PFN entry now */
         MiInitializePfn(PageFrameIndex, PointerPte, 1);

reactos/ntoskrnl/mm/ARM3/pfnlist.c

@@ -30,12 +30,11 @@ do { \
 #define ASSERT_LIST_INVARIANT(x)
 #endif
 
-#define ARM3_COLORS 1
-
 /* GLOBALS ********************************************************************/
 
 BOOLEAN MmDynamicPfn;
 BOOLEAN MmMirroring;
+ULONG MmSystemPageColor;
 
 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
@@ -80,11 +79,9 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
     PFN_NUMBER OldFlink, OldBlink;
     PMMPFNLIST ListHead;
     MMLISTS ListName;
-#ifdef ARM3_COLORS    
     ULONG Color;
     PMMCOLOR_TABLES ColorTable;
     PMMPFN Pfn1;
-#endif
 
     /* Make sure the PFN lock is held */
     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
@@ -131,7 +128,7 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
         /* Set the list head's backlink instead */
         ListHead->Flink = OldFlink;
     }
-#ifdef ARM3_COLORS    
+
     /* Get the page color */
     OldBlink = MiGetPfnEntryIndex(Entry);
     Color = OldBlink & MmSecondaryColorMask;
@@ -185,7 +182,7 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
 
     /* ReactOS Hack */
     Entry->OriginalPte.u.Long = 0;
-#endif    
+
     /* We are not on a list anymore */
     Entry->u1.Flink = Entry->u2.Blink = 0;
     ASSERT_LIST_INVARIANT(ListHead);
@@ -219,9 +216,8 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex,
     MMLISTS ListName;
     PFN_NUMBER OldFlink, OldBlink;
     ULONG OldColor, OldCache;
-#ifdef ARM3_COLORS    
     PMMCOLOR_TABLES ColorTable;
-#endif
+
     /* Make sure PFN lock is held */
     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
     ASSERT(Color < MmSecondaryColors);
@@ -280,7 +276,7 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex,
     Pfn1->u3.e2.ShortFlags = 0;
     Pfn1->u3.e1.PageColor = OldColor;
     Pfn1->u3.e1.CacheAttribute = OldCache;
-#ifdef ARM3_COLORS    
+
     /* Get the first page on the color list */
     ASSERT(Color < MmSecondaryColors);
     ColorTable = &MmFreePagesByColor[ListName][Color];
@@ -306,7 +302,7 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 
     /* ReactOS Hack */
     Pfn1->OriginalPte.u.Long = 0;
-#endif
+
     /* See if we hit any thresholds */
     if (MmAvailablePages == MmHighMemoryThreshold)
     {
@@ -340,9 +336,8 @@ MiRemoveAnyPage(IN ULONG Color)
     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
     ASSERT(MmAvailablePages != 0);
     ASSERT(Color < MmSecondaryColors);
-    
+#if 0
     /* Check the colored free list */
-#if 0 // Enable when using ARM3 database */
     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
     if (PageIndex == LIST_HEAD)
     {
@@ -368,11 +363,10 @@ MiRemoveAnyPage(IN ULONG Color)
                     ASSERT(MmZeroedPageListHead.Total == 0);
                 }
             }
-#if 0 // Enable when using ARM3 database */
+#if 0
         }
     }
 #endif
-
     /* Remove the page from its list */
     PageIndex = MiRemovePageByColor(PageIndex, Color);
 
@@ -403,7 +397,7 @@ MiRemoveZeroPage(IN ULONG Color)
     ASSERT(Color < MmSecondaryColors);
 
     /* Check the colored zero list */
-#if 0 // Enable when using ARM3 database */
+#if 0
     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
     if (PageIndex == LIST_HEAD)
     {
@@ -414,9 +408,10 @@ MiRemoveZeroPage(IN ULONG Color)
         Color = PageIndex & MmSecondaryColorMask;
         if (PageIndex == LIST_HEAD)
         {
+            /* This means there's no zero pages, we have to look for free ones */
             ASSERT(MmZeroedPageListHead.Total == 0);
             Zero = TRUE;
-#if 0 // Enable when using ARM3 database */
+#if 0
             /* Check the colored free list */
             PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
             if (PageIndex == LIST_HEAD)
@@ -432,13 +427,14 @@ MiRemoveZeroPage(IN ULONG Color)
                     /* FIXME: Should check the standby list */
                     ASSERT(MmZeroedPageListHead.Total == 0);
                 }
-#if 0 // Enable when using ARM3 database */
+#if 0
             }
 #endif
         }
-#if 0 // Enable when using ARM3 database */
+#if 0
     }
 #endif
+
     /* Sanity checks */
     Pfn1 = MiGetPfnEntry(PageIndex);
     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
@@ -468,11 +464,10 @@ MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
     PMMPFNLIST ListHead;
     PFN_NUMBER LastPage;
     PMMPFN Pfn1;
-#ifdef ARM3_COLORS    
     ULONG Color;
     PMMPFN Blink;
     PMMCOLOR_TABLES ColorTable;
-#endif
+
     /* Make sure the page index is valid */
     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
     ASSERT((PageFrameIndex != 0) &&
@@ -537,7 +532,7 @@ MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
         /* Otherwise check if we reached the high threshold and signal the event */
         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
     }
-#ifdef ARM3_COLORS    
+
     /* Get the page color */
     Color = PageFrameIndex & MmSecondaryColorMask;
 
@@ -571,7 +566,7 @@ MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 
     /* And increase the count in the colored list */
     ColorTable->Count++;
-#endif
+
     /* Notify zero page thread if enough pages are on the free list now */
     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
     {
@@ -590,10 +585,9 @@ MiInsertPageInList(IN PMMPFNLIST ListHead,
     PFN_NUMBER Flink;
     PMMPFN Pfn1, Pfn2;
     MMLISTS ListName;
-#ifdef ARM3_COLORS
     PMMCOLOR_TABLES ColorHead;
     ULONG Color;
-#endif
+
     /* For free pages, use MiInsertPageInFreeList */
     ASSERT(ListHead != &MmFreePageListHead);
 
@@ -657,7 +651,7 @@ MiInsertPageInList(IN PMMPFNLIST ListHead,
         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
     }
 
-#ifdef ARM3_COLORS
+    /* Sanity checks */
     ASSERT(ListName == ZeroedPageList);
     ASSERT(Pfn1->u4.InPageError == 0);
 
@@ -695,7 +689,6 @@ MiInsertPageInList(IN PMMPFNLIST ListHead,
 
     /* One more paged on the colored list */
     ColorHead->Count++;
-#endif
 }
 
 VOID
@@ -783,7 +776,7 @@ MiAllocatePfn(IN PMMPTE PointerPte,
     /* Grab a page */
     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
-    PageFrameIndex = MiRemoveAnyPage(0);
+    PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
 
     /* Write the software PTE */
     MI_WRITE_INVALID_PTE(PointerPte, TempPte);

reactos/ntoskrnl/mm/ARM3/pool.c

@@ -473,7 +473,7 @@ MiAllocatePoolPages(IN POOL_TYPE PoolType,
                 ASSERT(PointerPte->u.Hard.Valid == 0);
 
                 /* Request a page */
-                PageFrameNumber = MiRemoveAnyPage(0);
+                PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
                 TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 
 #if (_MI_PAGING_LEVELS >= 3)
@@ -768,7 +768,7 @@ MiAllocatePoolPages(IN POOL_TYPE PoolType,
     do
     {
         /* Allocate a page */
-        PageFrameNumber = MiRemoveAnyPage(0);
+        PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
 
         /* Get the PFN entry for it and fill it out */
         Pfn1 = MiGetPfnEntry(PageFrameNumber);