Merge branch 'master' into netstack

Docs/articles/Kernel/MemoryManagement.md

@@ -45,6 +45,10 @@ The garbage collector has to be manually triggerd using the call `int Heap.Colle
 
 Note that the GC does not track objects only pointed to by pointers. To ensure that the GC nevertheless does not incorrectly free objects, you can use `void GCImplementation.IncRootCount(ushort* aPtr)` to manually increase the references of your object by 1. Once you no longer need the object you can use `void GCImplementation.DecRootCount(ushort* aPtr)` to remove the manual reference, which allows the next `Heap.Collect` call to free the object. 
 
+## Automatically Trigger Garbage Collection
+
+When `RAT.MinFreePages` is set to a positive value and the number of free pages (as tracked by `RAT.FreePageCount`) drops below this value, on page allocation `Heap.Collect` will automatically be called. Each time this happens the value in `RAT.GCTriggered` is incremented by one.
+
 ### Internals
 
 The garbage collector uses the tracing approach, which means that during collection a graph of all reachable objects is created and all non-discovered objects are freed. The garbage collector will only check objects on pages which have a type where the `GCManaged` bit is set. The graph is created by starting from "root" objects which are either stored in static variables or part of the current stack. Each of these objects is "marked" and all objects referenced by this object are recursivly also "marked" and "swept". This is done using the methods `void Heap.MarkAndSweepObject(void* aPtr)` for objects and `void Heap.SweepTypedObject(uint* obj, uint type)` for structures. For this to work each allocated object holds a 1bit flag if the object was discovered during the marking phase and a 7bit value counter for the number of static references it has. The number of static references an object has is updated using `void GCImplementation.IncRootCount(ushort* aPtr)` and similar methods, which are called from the Stsfld opcode.

Test.sln

@@ -106,8 +106,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Core.Memory.Test", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Core_Asm", "source\Cosmos.Core_Asm\Cosmos.Core_Asm.csproj", "{B7077A34-D7F0-4422-BE7C-65DF26C65489}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TheRingMaster", "source\TheRingMaster\TheRingMaster.csproj", "{3DD192AF-2D72-449F-936C-ED8734225B18}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spruce", "..\XSharp\source\Spruce\Spruce.csproj", "{FF46829E-B612-4D36-80BE-ED04521AD91A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Compiler.Tests.TypeSystem", "Tests\Kernels\Cosmos.Compiler.Tests.TypeSystem\Cosmos.Compiler.Tests.TypeSystem.csproj", "{D21A7C6C-A696-4EC3-84EB-70700C1E3B34}"
@@ -867,7 +865,6 @@ Global
 		{FB23BD72-AEC3-485E-B86C-8E7DB0B3BB9B} = {29EEC029-6A2B-478A-B6E5-D63A91388ABA}
 		{408E5ACC-EA9A-41E8-AA95-514C5F47BD34} = {52D81759-C7CC-427F-8C96-89CA10C914B5}
 		{B7077A34-D7F0-4422-BE7C-65DF26C65489} = {04B18FFC-8EA0-4E9F-9E1B-478527B19AFA}
-		{3DD192AF-2D72-449F-936C-ED8734225B18} = {C286932C-3F6D-47F0-BEEF-26843D1BB11B}
 		{FF46829E-B612-4D36-80BE-ED04521AD91A} = {E9CD521E-C386-466D-B5F7-A5EB19A61625}
 		{D21A7C6C-A696-4EC3-84EB-70700C1E3B34} = {ECEA7778-E786-4317-90B9-A2D4427CB91C}
 		{0DF97CAC-220B-4DAD-B397-42E394255763} = {ECEA7778-E786-4317-90B9-A2D4427CB91C}

Tests/Cosmos.Core.Memory.Test/RATTest.cs

@@ -44,10 +44,16 @@ public unsafe void RATMethods()
 
                 RAT.Init(xPtr, (uint)xRAM.Length);
 
+                uint freePageCount = RAT.FreePageCount;
+                Assert.IsTrue(freePageCount < RAT.TotalPageCount);
+                Assert.AreEqual(freePageCount, RAT.GetPageCount((byte)RAT.PageType.Empty));
+
                 var largePage = RAT.AllocPages(RAT.PageType.HeapLarge, 3);
                 Assert.AreEqual(RAT.PageType.HeapLarge, RAT.GetPageType(largePage));
                 Assert.AreEqual(RAT.GetFirstRATIndex(largePage), RAT.GetFirstRATIndex((byte*)largePage + 20));
                 Assert.AreEqual(RAT.PageType.HeapLarge, RAT.GetPageType((byte*)largePage + RAT.PageSize));
+
+                Assert.AreEqual(RAT.FreePageCount, freePageCount - 3);
             }
         }
 

Tests/Kernels/MemoryOperationsTest/Kernel.cs

@@ -157,6 +157,7 @@ static unsafe void TestMemoryBlock(MemoryBlock memoryBlock)
             memoryBlock.Read32(read);
             Assert.AreEqual(values, read, "Using Fill(int, int, int) works");
         }
+
         static unsafe void TestRealloc()
 		{
             // Allocate initial pointer and fill with value 32
@@ -174,15 +175,40 @@ static unsafe void TestRealloc()
             Assert.AreEqual(aPtr[16], 0, "Expected value 0 not found at the end of aPtr.");
 		}
 
+        static unsafe void TestMemoryManager()
+        {
+            uint freePageCount = RAT.FreePageCount;
+            Assert.IsTrue(freePageCount < RAT.TotalPageCount, "Number of free pages is less than total number of pages");
+            Assert.AreEqual(freePageCount, RAT.GetPageCount((byte)RAT.PageType.Empty), "GetPageCount and FreePageCount get different number of free pages");
+            void* pointer = RAT.AllocPages(RAT.PageType.HeapLarge, 3);
+            Assert.AreEqual(RAT.FreePageCount, freePageCount - 3, "Allocating three pages reduces number of free pages by three");
+            RAT.Free(RAT.GetFirstRATIndex(pointer));
+            Assert.AreEqual(RAT.FreePageCount, freePageCount, "Freeing the allocated pages correctly updates the number of free pages");
+        }
+
+        void TestAutomaticGCCollect()
+        {
+            Assert.AreEqual(RAT.GCTriggered, 0, "Before Enabling GC hasnt been triggered automatically");
+            RAT.MinFreePages = (int)RAT.FreePageCount - 10;
+            while (RAT.GCTriggered == 0)
+            {
+                Console.WriteLine($"Free: {RAT.FreePageCount} Min: {RAT.MinFreePages} GC Triggered: {RAT.GCTriggered} Objects: {HeapSmall.GetAllocatedObjectCount()}");
+            }
+            Assert.Succeed("GC can be triggered automatically");
+            RAT.MinFreePages = -1;
+        }
+
         protected override void Run()
         {
             try
             {
+                TestMemoryManager();
                 TestCopy();
                 TestMemoryBlock(new MemoryBlock(0x60000, 128)); //we are testing in SVGA video memory which should not be in use
                 TestManagedMemoryBlock(new ManagedMemoryBlock(128));
                 TestRealloc();
                 SpanTest.Execute();
+                TestAutomaticGCCollect();
                 TestController.Completed();
             }
             catch (Exception e)

source/Cosmos.Build.Tasks/Cosmos.Build.Tasks.csproj

@@ -2,7 +2,7 @@
 
     <PropertyGroup>
         <TargetFramework>net48</TargetFramework>
-        <RuntimeIdentifier>win-x86</RuntimeIdentifier>
+        <RuntimeIdentifier>win-x64</RuntimeIdentifier>
         <PackageId>Cosmos.Build</PackageId>
         <PackageDescription>Cosmos build system.</PackageDescription>
         <IsMSBuildExtensionProject>True</IsMSBuildExtensionProject>

source/Cosmos.Core/Memory/RAT.cs

@@ -93,7 +93,23 @@ public enum PageType : byte
         /// Number of pages in the heap.
         /// </summary>
         /// <remarks>Calculated from mSize.</remarks>
-        static public uint TotalPageCount;
+        public static uint TotalPageCount;
+
+        /// <summary>
+        /// Number of pages which are currently not in use
+        /// </summary>
+        public static uint FreePageCount;
+
+        /// <summary>
+        /// If number of free pages drops below this number, we trigger the GC.Collect automatically
+        /// If set to -1 it is disabled
+        /// </summary>
+        public static int MinFreePages = -1;
+
+        /// <summary>
+        /// Number of the times the GC has been triggered automatically
+        /// </summary>
+        public static uint GCTriggered = 0;
 
         /// <summary>
         /// Pointer to the RAT.
@@ -132,6 +148,7 @@ public static void Init(byte* aStartPtr, uint aSize)
             RamSize = aSize;
             HeapEnd = aStartPtr + aSize;
             TotalPageCount = aSize / PageSize;
+            FreePageCount = TotalPageCount;
 
             // We need one status byte for each block.
             // Intel blocks are 4k (10 bits). So for 4GB, this means
@@ -150,6 +167,9 @@ public static void Init(byte* aStartPtr, uint aSize)
             {
                 *p = (byte)PageType.RAT;
             }
+            // Remove pages needed for RAT table from count
+            FreePageCount -= xRatPageCount;
+
             Heap.Init();
         }
 
@@ -192,6 +212,12 @@ public static uint GetPageCount(byte aType = 0)
         /// <returns>A pointer to the first page on success, null on failure.</returns>
         public static void* AllocPages(PageType aType, uint aPageCount = 1)
         {
+            if (MinFreePages > FreePageCount)
+            {
+                Heap.Collect();
+                GCTriggered++;
+            }
+
             byte* xPos = null;
 
             // Could combine with an external method or delegate, but will slow things down
@@ -241,6 +267,10 @@ public static uint GetPageCount(byte aType = 0)
                     *p = (byte)PageType.Extension;
                 }
                 CPU.ZeroFill((uint)xResult, PageSize * aPageCount);
+
+                // Decrement free page count
+                FreePageCount -= aPageCount;
+
                 return xResult;
             }
             return null;
@@ -295,13 +325,15 @@ public static void Free(uint aPageIdx)
         {
             byte* p = mRAT + aPageIdx;
             *p = (byte)PageType.Empty;
+            FreePageCount++;
             for (; p < mRAT + TotalPageCount; )
             {
                 if (*++p != (byte)PageType.Extension)
                 {
                     break;
                 }
                 *p = (byte)PageType.Empty;
+                FreePageCount++;
             }
         }
     }