Merge remote-tracking branch 'origin/dotnet8' into dotnet8

Build/VMWare/Workstation/Cosmos.vmx

@@ -51,8 +51,11 @@ serial0.pipe.endPoint = "client"
 serial0.tryNoRxLoss = "TRUE"
 sound.present = "TRUE"
 ethernet0.present = "TRUE"
-ethernet0.wakeOnPcktRcv = "FALSE"
+ethernet0.connectionType = "nat"
 ethernet0.addressType = "generated"
+ethernet0.generatedAddress = "00:0c:29:0c:c9:44"
+ethernet0.generatedAddressOffset = "0"
+ethernet0.pciSlotNumber = "34"
 usb.present = "TRUE"
 usb.generic.allowHID = "TRUE"
 ehci.present = "TRUE"

Docs/articles/Kernel/MemoryManagement.md

@@ -35,7 +35,16 @@ The RAT is managed through the `RAT` class. Pages are allocated via `void* RAT.A
 
 The Heap itself is managed by the `Heap` class. It contains the mechanism to allocate (`byte* Heap.Alloc(uint aSize)`), re-allocate ('byte* Heap.Realloc(byte* aPtr, uint newSize)') and free (`void Heap.Free(void* aPtr)`) objects of various sizes. Objects are seperated by size in bytes into Small (Smaller than 1/4 Page), Medium (Smaller than 1 Page) and Large (Larger than 1 Page). Currently Medium and Large objects are managed the same way using the methods in `HeapLarge` which do little more than allocating/freeing the necessary number of pages. Small objects are managed differently in `HeapSmall`. 
 
-Small Objects are managed using the SMT (Size Map Table), which is initalised using `void HeapSmall.InitSMT(uint aMaxItemSize)`. The basic idea of the SMT is to allocate objects of similar sizes on the same page. The SMT grows dynamically as required. The SMT is made up of a series of pages, each of which contains a series of `RootSMTBlock` each of which link to a chain of `SMTBlock`. The `RootSMTBlock` can be thought of as column headers and the `SMTBlock` as the elements stored in the column. The `RootSMTBlock` are a linked list, each containing the maximum object size stored in its pages, the location of the first `SMTBlock` for this size, and the location of the next `RootSMTBlock`. The list is in ascending order of size, so that the smallest large enough `RootSMTBlock` is found first. A `SMTBlock` contains a pointer to the actual page where objects are stored, how much space is left on that page, and a pointer to the next `SMTBlock`. If every `SMTBlock` for a certain size is full, a new `SMTBlock` is allocated. The page linked to by the `SMTBlock` is split into an array of spaces, each large enough to allocate an object of maximum size with header, which can be iterated through via index and fixed size when allocating. Each object allocated on the `HeapSmall` has a header of 2 `ushort`, the first one storing the actual size of the object and the second, the GC status of the object.
+Small Objects are managed using the SMT (Size Map Table), which is initalised using `void HeapSmall.InitSMT(uint aMaxItemSize)`.
+The basic idea of the SMT is to allocate objects of similar sizes on the same page. The SMT grows dynamically as required.
+The SMT is made up of a series of pages, each of which contains a series of `RootSMTBlock` each of which link to a chain of `SMTBlock`.
+The `RootSMTBlock` can be thought of as column headers and the `SMTBlock` as the elements stored in the column.
+The `RootSMTBlock` are a linked list, each containing the maximum object size stored in its pages, the location of the first `SMTBlock` for this size, and the location of the next `RootSMTBlock`.
+The list is in ascending order of size, so that the smallest large enough `RootSMTBlock` is found first.
+A `SMTBlock` contains a pointer to the actual page where objects are stored, how much space is left on that page, and a pointer to the next `SMTBlock`.
+If every `SMTBlock` for a certain size is full, a new `SMTBlock` is allocated.
+The page linked to by the `SMTBlock` is split into an array of spaces, each large enough to allocate an object of maximum size with header, which can be iterated through via index and fixed size when allocating.
+Each object allocated on the `HeapSmall` has a header of 2 `ushort`, the first one storing the actual size of the object and the second, the GC status of the object.
 
 ## Garbage Collection
 
@@ -45,6 +54,10 @@ The garbage collector has to be manually triggerd using the call `int Heap.Colle
 
 Note that the GC does not track objects only pointed to by pointers. To ensure that the GC nevertheless does not incorrectly free objects, you can use `void GCImplementation.IncRootCount(ushort* aPtr)` to manually increase the references of your object by 1. Once you no longer need the object you can use `void GCImplementation.DecRootCount(ushort* aPtr)` to remove the manual reference, which allows the next `Heap.Collect` call to free the object. 
 
+`Heap.Collect` only cleans up the objects which are no longer used but will leave behind empty pages in the SMT.
+These pages can be cleaned up using `HeapSmall.PruneSMT` which will return the number of pages it freed.
+Note that if in future elements are reallocated, this will cause new pages in the SMT to be allocated again, so using this too often may not be useful.
+
 ## Automatically Trigger Garbage Collection
 
 When `RAT.MinFreePages` is set to a positive value and the number of free pages (as tracked by `RAT.FreePageCount`) drops below this value, on page allocation `Heap.Collect` will automatically be called. Each time this happens the value in `RAT.GCTriggered` is incremented by one.

Docs/articles/Kernel/VFS.md

@@ -45,7 +45,7 @@ Console.WriteLine("Available Free Space: " + available_space);
 You have probably noticed the "0:\" argument passed to this function, this is the id of the drive that we want to get available free space of.
 Cosmos using DOS drive naming system and this is why we use "0".
 
-**Attention**: Typing "0:/" instead of "0:\" might lead to errors, you've been warned.
+**Attention**: Typing "0:/" instead of "0:\\" might lead to errors, you've been warned.
 
 ## Get file system type
 

Tests/Cosmos.TestRunner.Core/Engine.Run.cs

@@ -1,6 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading;
 
 namespace Cosmos.TestRunner.Core
 {
@@ -49,22 +53,34 @@ private bool ExecuteKernel(
 
             RunTask("MakeISO", () => MakeIso(xObjectFile, xIsoFile));
 
-            switch (configuration.RunTarget)
+            Console.WriteLine("assemblyFileName=" + assemblyFileName);
+
+            if (assemblyFileName.EndsWith("NetworkTest.dll"))
+            {
+                var serverThread = new Thread(StartTcpServer);
+                serverThread.Start();
+
+                RunTask("RunISO", () => RunIsoInVMware(xIsoFile, xHarddiskPath));
+            }
+            else
             {
-                case RunTargetEnum.Bochs:
-                    RunTask("RunISO", () => RunIsoInBochs(xIsoFile, xHarddiskPath, workingDirectory));
-                    break;
-                case RunTargetEnum.Qemu:
-                    RunTask("RunISO", () => RunIsoInQemu(xIsoFile, xHarddiskPath, workingDirectory));
-                    break;
-                case RunTargetEnum.VMware:
-                    RunTask("RunISO", () => RunIsoInVMware(xIsoFile, xHarddiskPath));
-                    break;
-                case RunTargetEnum.HyperV:
-                    RunTask("RunISO", () => RunIsoInHyperV(xIsoFile, xHarddiskPath));
-                    break;
-                default:
-                    throw new ArgumentOutOfRangeException("RunTarget " + configuration.RunTarget + " not implemented!");
+                switch (configuration.RunTarget)
+                {
+                    case RunTargetEnum.Bochs:
+                        RunTask("RunISO", () => RunIsoInBochs(xIsoFile, xHarddiskPath, workingDirectory));
+                        break;
+                    case RunTargetEnum.Qemu:
+                        RunTask("RunISO", () => RunIsoInQemu(xIsoFile, xHarddiskPath, workingDirectory));
+                        break;
+                    case RunTargetEnum.VMware:
+                        RunTask("RunISO", () => RunIsoInVMware(xIsoFile, xHarddiskPath));
+                        break;
+                    case RunTargetEnum.HyperV:
+                        RunTask("RunISO", () => RunIsoInHyperV(xIsoFile, xHarddiskPath));
+                        break;
+                    default:
+                        throw new ArgumentOutOfRangeException("RunTarget " + configuration.RunTarget + " not implemented!");
+                }
             }
 
             OutputHandler.ExecuteKernelEnd(assemblyFileName);
@@ -94,5 +110,102 @@ private void RunTask(string aTaskName, Action aAction)
                 OutputHandler.TaskEnd(aTaskName);
             }
         }
+
+        private void StartTcpServer()
+        {
+            var listener = new TcpListener(IPAddress.Loopback, 12345);
+            listener.Start();
+
+            Console.WriteLine("TCP server started in a new thread, waiting connection from test kernel...");
+
+            IPEndPoint localEndPoint = listener.LocalEndpoint as IPEndPoint;
+            Console.WriteLine($"IP: {localEndPoint.Address}, Port: {localEndPoint.Port}");
+
+            var client = listener.AcceptTcpClient();
+            var remoteEndPoint = client.Client.RemoteEndPoint as IPEndPoint;
+            var clientIPAddress = remoteEndPoint.Address;
+            Console.WriteLine("Test kernel connected! Beginning tests...");
+
+            IPAddress remoteIPAddress = null;
+
+            using (NetworkStream stream = client.GetStream())
+            {
+                // Test 1: Send simple message
+                string testMessage = "Hello from the testrunner!";
+                byte[] messageBytes = Encoding.ASCII.GetBytes(testMessage);
+                stream.Write(messageBytes, 0, messageBytes.Length);
+                Console.WriteLine($"Sent: {testMessage}");
+
+                // Test 2: Receive a message from kernel
+                byte[] bufferIp = new byte[1024];
+                int bytesIpRead = stream.Read(bufferIp, 0, bufferIp.Length);
+                string ip = Encoding.ASCII.GetString(bufferIp, 0, bytesIpRead);
+                remoteIPAddress = IPAddress.Parse(ip);
+                Console.WriteLine($"Received: {ip}");
+
+                // Test 2.2: Receive a message from kernel
+                byte[] buffer = new byte[1024];
+                int bytesRead = stream.Read(buffer, 0, buffer.Length);
+                string receivedMessage = Encoding.ASCII.GetString(buffer, 0, bytesRead);
+                Console.WriteLine($"Received: {receivedMessage}");
+
+                // Test 3: Send back the received message, capitalized
+                string replyMessage = receivedMessage.ToUpper();
+                byte[] replyBytes = Encoding.ASCII.GetBytes(replyMessage);
+                stream.Write(replyBytes, 0, replyBytes.Length);
+                Console.WriteLine($"Sent: {replyMessage}");
+
+                // Test 4: Receive a big packet from kernel to test TCP sequencing
+                byte[] buffer2 = new byte[6000];
+                int totalBytesRead = 0;
+
+                while (totalBytesRead < 6000)
+                {
+                    totalBytesRead += stream.Read(buffer2, totalBytesRead, 6000 - totalBytesRead);
+                }
+
+                // Test 5: Send back the received message
+                stream.Write(buffer2, 0, buffer2.Length);
+                Console.WriteLine($"Sent: {replyMessage}");
+            }
+
+            client.Close();
+            listener.Stop();
+
+            ConnectToTcpServer(remoteIPAddress);
+        }
+
+        private void ConnectToTcpServer(IPAddress ip)
+        {
+            var xClient = new TcpClient();
+
+            Console.WriteLine("Attempting to connect to the kernel...");
+
+            try
+            {
+                // Test 6: Test TCPListener implementation
+                xClient.Connect(ip, 4343);
+                Console.WriteLine("Connected to the kernel!");
+
+                using (NetworkStream stream = xClient.GetStream())
+                {
+                    byte[] buffer = new byte[1024];
+                    int bytesRead = stream.Read(buffer, 0, buffer.Length);
+                    string receivedMessage = Encoding.ASCII.GetString(buffer, 0, bytesRead);
+                    Console.WriteLine($"Received: {receivedMessage}");
+
+                    string testMessage = "Hello from the testrunner again!";
+                    byte[] messageBytes = Encoding.ASCII.GetBytes(testMessage);
+                    stream.Write(messageBytes, 0, messageBytes.Length);
+                    Console.WriteLine($"Sent: {testMessage}");
+                }
+
+                xClient.Close();
+            }
+            catch (SocketException ex)
+            {
+                Console.WriteLine($"Failed to connect to the kernel. Error: {ex.Message}");
+            }
+        }
     }
 }

Tests/Cosmos.TestRunner.Full/TestKernelSets.cs

@@ -30,7 +30,9 @@ public static IEnumerable<Type> GetStableKernelTypes()
 
             //yield return typeof(KernelGen3.Boot);
             yield return typeof(GraphicTest.Kernel);
-            yield return typeof(NetworkTest.Kernel);
+
+            // Disable network tests due to our self hosted CI/CD limitations (VPS currently doesn't support VMWare with its CPU)
+            //yield return typeof(NetworkTest.Kernel);
             yield return typeof(AudioTests.Kernel);
             // Please see the notes on the kernel itself before enabling it
             //yield return typeof(ConsoleTest.Kernel);

Tests/Kernels/Cosmos.Compiler.Tests.TypeSystem/Kernel.cs

@@ -140,6 +140,18 @@ private unsafe void TestGarbageCollector()
             StaticTestClass.B.FieldA = 10;
             collected = Heap.Collect();
             Assert.AreEqual(0, collected, "Storing elements in static class keeps them referenced");
+
+            for (int i = 0; i < 10_000; i++)
+            {
+                _ = new object();
+            }
+            Heap.Collect();
+            uint heapSmallPages = RAT.GetPageCount((byte)RAT.PageType.HeapSmall);
+            int freed = HeapSmall.PruneSMT();
+            uint afterPrune = RAT.GetPageCount((byte)RAT.PageType.HeapSmall);
+            Assert.IsTrue(heapSmallPages >= afterPrune, "Running PruneSMT does not increase the number of pages in use");
+            Assert.AreEqual(freed, heapSmallPages - afterPrune, "PruneSMT returns the correct number of pages freed");
+
         }
 
         #region Test Methods