page: Add gfp_flags to memory alloc interfaces

Also add GFP_KERNEL for default gfp flags.

This work was derived from KTSAN work, hence why some changes don't make
much sense out-of-context.
Since KTSAN is not stable and will take a good while to become so, let's
queue this up already, as an important patch.

Signed-off-by: Pedro Falcato <[email protected]>

kernel/arch/riscv64/scheduler.cpp

@@ -128,7 +128,7 @@ thread *sched_spawn_thread(registers_t *regs, unsigned int flags, void *tp)
     new_thread->refcount = 1;
 
     new_thread->kernel_stack =
-        static_cast<uintptr_t *>(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE));
+        static_cast<uintptr_t *>(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE, GFP_KERNEL));
 
     if (!new_thread->kernel_stack)
     {

kernel/arch/x86_64/thread.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016 - 2021 Pedro Falcato
+ * Copyright (c) 2016 - 2023 Pedro Falcato
  * This file is part of Onyx, and is released under the terms of the MIT License
  * check LICENSE at the root directory for more information
  *
@@ -170,6 +170,7 @@ extern "C" void thread_finish_destruction(void *___thread)
 thread *sched_spawn_thread(registers_t *regs, unsigned int flags, void *fs)
 {
     thread *new_thread = new thread;
+    uintptr_t *thr_stack_alloc = nullptr;
 
     if (!new_thread)
         return NULL;
@@ -211,9 +212,10 @@ thread *sched_spawn_thread(registers_t *regs, unsigned int flags, void *fs)
 
     new_thread->refcount = 1;
 
-    new_thread->kernel_stack =
-        static_cast<uintptr_t *>(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE));
+    thr_stack_alloc =
+        static_cast<uintptr_t *>(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE, GFP_KERNEL));
 
+    new_thread->kernel_stack = thr_stack_alloc;
     if (!new_thread->kernel_stack)
     {
         goto error;

kernel/drivers/ahci/ahci.cpp

@@ -725,7 +725,7 @@ bool ahci_io_queue::init(ahci_hba_memory_regs_t *hba, ahci_port_t *port, ahci_po
         goto error;
 
     /* The fisb is 1024 bytes in size, with 1024 alignment */
-    virtual_fisb = vmalloc(1, VM_TYPE_REGULAR, VM_WRITE | VM_READ);
+    virtual_fisb = vmalloc(1, VM_TYPE_REGULAR, VM_WRITE | VM_READ, GFP_KERNEL);
 
     if (!virtual_fisb)
         goto error;

kernel/include/onyx/kunit.h

@@ -28,7 +28,7 @@ struct test
     constexpr test(const char* name) : name_{name}
     {
     }
-    virtual void do_test();
+    virtual void do_test() = 0;
 };
 
 }; // namespace onx

kernel/include/onyx/mm/pool.hpp

@@ -1,7 +1,9 @@
 /*
- * Copyright (c) 2020 Pedro Falcato
+ * Copyright (c) 2020 - 2023 Pedro Falcato
  * This file is part of Onyx, and is released under the terms of the MIT License
  * check LICENSE at the root directory for more information
+ *
+ * SPDX-License-Identifier: MIT
  */
 
 #ifndef _ONYX_MM_POOL_HPP
@@ -310,7 +312,7 @@ class memory_pool
         seg->~memory_pool_segment<T, using_vm>();
     }
 
-    NO_ASAN bool expand_pool()
+    NO_ASAN bool expand_pool(unsigned int gfp_flags)
     {
         // std::cout << "Expanding pool.\n";
         auto allocation_size = memory_pool_segment<T, using_vm>::memory_pool_size();
@@ -320,17 +322,18 @@ class memory_pool
 
         if constexpr (using_vm)
         {
-            void *vmalloc_seg =
-                vmalloc(allocation_size >> PAGE_SHIFT, VM_TYPE_REGULAR, VM_READ | VM_WRITE);
+            void *vmalloc_seg = vmalloc(allocation_size >> PAGE_SHIFT, VM_TYPE_REGULAR,
+                                        VM_READ | VM_WRITE, gfp_flags);
             if (!vmalloc_seg)
                 return false;
             seg.set_vmalloc_seg(vmalloc_seg);
             address = vmalloc_seg;
         }
         else
         {
-            struct page *pages = alloc_pages(allocation_size >> PAGE_SHIFT,
-                                             PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS);
+            struct page *pages =
+                alloc_pages(allocation_size >> PAGE_SHIFT,
+                            PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS | gfp_flags);
             if (!pages)
                 return false;
             seg.set_pages(pages);
@@ -457,13 +460,13 @@ class memory_pool
     }
 
     NO_ASAN
-    T *allocate()
+    T *allocate(unsigned int gfp_flags = GFP_KERNEL)
     {
         scoped_lock<spinlock, usable_onirq> guard{lock};
 
         while (!free_chunk_head)
         {
-            if (!expand_pool())
+            if (!expand_pool(gfp_flags))
             {
                 // std::cout << "mmap failed\n";
                 return nullptr;

kernel/include/onyx/page.h

@@ -158,13 +158,24 @@ void page_init(size_t memory_size, unsigned long maxpfn);
 
 unsigned int page_hash(uintptr_t p);
 struct page *phys_to_page(uintptr_t phys);
+
+/**
+ * @brief Retrieve the struct page from a physical address
+ * This may fail if phys > maxpfn
+ * @param phys Physical address
+ * @return Struct page, or NULL if > maxpfn
+ */
+struct page *phys_to_page_mayfail(uintptr_t phys);
 struct page *page_add_page(void *paddr);
 struct page *page_add_page_late(void *paddr);
 
-#define PAGE_ALLOC_CONTIGUOUS     (1 << 0)
-#define PAGE_ALLOC_NO_ZERO        (1 << 1)
-#define PAGE_ALLOC_4GB_LIMIT      (1 << 2)
-#define PAGE_ALLOC_INTERNAL_DEBUG (1 << 3)
+#define PAGE_ALLOC_CONTIGUOUS          (1 << 0)
+#define PAGE_ALLOC_NO_ZERO             (1 << 1)
+#define PAGE_ALLOC_4GB_LIMIT           (1 << 2)
+#define PAGE_ALLOC_INTERNAL_DEBUG      (1 << 3)
+#define PAGE_ALLOC_NO_SANITIZER_SHADOW (1 << 4)
+
+#define GFP_KERNEL 0
 
 static inline bool __page_should_zero(unsigned long flags)
 {

kernel/include/onyx/vm.h

@@ -351,9 +351,10 @@ void *vm_get_fallback_pgd();
  * @param pages The number of pages.
  * @param type The type of allocation.
  * @param perms The permissions on the allocation.
+ * @param gfp_flags GFP flags
  * @return A pointer to the new allocation, or NULL with errno set on failure.
  */
-void *vmalloc(size_t pages, int type, int perms);
+void *vmalloc(size_t pages, int type, int perms, unsigned int gfp_flags);
 
 /**
  * @brief Frees a region of memory previously allocated by vmalloc.

kernel/kernel/compression.cpp

@@ -60,7 +60,7 @@ expected<unique_ptr<decompression_stream>, int> create_decompression_stream(
 
 bool decompress_bytestream::init(size_t len)
 {
-    buf = vmalloc(vm_size_to_pages(len), VM_TYPE_REGULAR, VM_WRITE | VM_READ);
+    buf = vmalloc(vm_size_to_pages(len), VM_TYPE_REGULAR, VM_WRITE | VM_READ, GFP_KERNEL);
     if (!buf)
         return false;
     this->len = len;

kernel/kernel/kcov.cpp

@@ -133,8 +133,8 @@ int kcov_init_trace(unsigned long nr_elems, struct file *f)
         return -EINVAL;
 
     // We use a vmalloc region and set up a VMO for it. This VMO is then mmap'd.
-    auto buffer =
-        (unsigned long *) vmalloc(vm_size_to_pages(size), VM_TYPE_REGULAR, VM_READ | VM_WRITE);
+    auto buffer = (unsigned long *) vmalloc(vm_size_to_pages(size), VM_TYPE_REGULAR,
+                                            VM_READ | VM_WRITE, GFP_KERNEL);
 
     if (!buffer)
         return -ENOMEM;

kernel/kernel/mm/page.cpp

@@ -51,6 +51,20 @@ struct page *phys_to_page(uintptr_t phys)
     return page_map + pfn - base_pfn;
 }
 
+/**
+ * @brief Retrieve the struct page from a physical address
+ * This may fail if phys > maxpfn
+ * @param phys Physical address
+ * @return Struct page, or NULL if > maxpfn
+ */
+struct page *phys_to_page_mayfail(uintptr_t phys)
+{
+    unsigned long pfn = phys >> PAGE_SHIFT;
+    if (pfn > maxpfn)
+        return nullptr;
+    return page_map + pfn - base_pfn;
+}
+
 extern unsigned char kernel_start;
 extern unsigned char kernel_end;
 uint64_t kernel_phys_offset = 0;

kernel/kernel/mm/slab.cpp

@@ -397,7 +397,7 @@ NO_ASAN static struct slab *kmem_cache_create_slab(struct slab_cache *cache, uns
     }
     else
     {
-        start = (char *) vmalloc(slab_size >> PAGE_SHIFT, VM_TYPE_HEAP, VM_READ | VM_WRITE);
+        start = (char *) vmalloc(slab_size >> PAGE_SHIFT, VM_TYPE_HEAP, VM_READ | VM_WRITE, flags);
         if (!start)
             return nullptr;
     }

kernel/kernel/mm/vmalloc.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Pedro Falcato
+ * Copyright (c) 2022 - 2023 Pedro Falcato
  * This file is part of Onyx, and is released under the terms of the MIT License
  * check LICENSE at the root directory for more information
  *
@@ -131,12 +131,13 @@ static memory_pool<vmalloc_region> pool;
  * @param start Start of the region
  * @param pages Number of pages of the region
  * @param perms Permissions
+ * @param gfp_flags GFP flags
  * @return Pointer to a vmalloc_region, or nullptr
  */
-struct vmalloc_region *vmalloc_insert_region(struct vmalloc_tree *tree, unsigned long start,
-                                             size_t pages, int perms)
+static struct vmalloc_region *vmalloc_insert_region(struct vmalloc_tree *tree, unsigned long start,
+                                                    size_t pages, int perms, unsigned int gfp_flags)
 {
-    auto reg = pool.allocate();
+    auto reg = pool.allocate(gfp_flags);
     if (!reg)
         return nullptr;
     reg->addr = start;
@@ -172,17 +173,19 @@ struct vmalloc_region *vmalloc_insert_region(struct vmalloc_tree *tree, unsigned
  * @param pages The number of pages.
  * @param type The type of allocation.
  * @param perms The permissions on the allocation.
+ * @param gfp_flags GFP flags
  * @return A pointer to the new allocation, or NULL with errno set on failure.
  */
-void *vmalloc(size_t pages, int type, int perms)
+void *vmalloc(size_t pages, int type, int perms, unsigned int gfp_flags)
 {
     scoped_lock g{vmalloc_tree.lock};
     auto start = vmalloc_allocate_base(&vmalloc_tree, 0, pages << PAGE_SHIFT);
 
     if (start + (pages << PAGE_SHIFT) > vmalloc_tree.start + vmalloc_tree.length)
         return errno = ENOMEM, nullptr;
 
-    auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, perms);
+    auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, perms,
+                                          gfp_flags | PAGE_ALLOC_NO_SANITIZER_SHADOW);
     if (!vmal_reg)
         return errno = ENOMEM, nullptr;
 
@@ -199,7 +202,7 @@ void *vmalloc(size_t pages, int type, int perms)
     }
 #endif
 
-    auto pgs = alloc_pages(pages, 0);
+    auto pgs = alloc_pages(pages, gfp_flags);
     if (!pgs)
     {
         delvmr();
@@ -336,7 +339,8 @@ void *mmiomap(void *phys, size_t size, size_t flags)
     if (start + (pages << PAGE_SHIFT) > vmalloc_tree.start + vmalloc_tree.length)
         return errno = ENOMEM, nullptr;
 
-    auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, flags);
+    auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, flags,
+                                          GFP_KERNEL | PAGE_ALLOC_NO_SANITIZER_SHADOW);
     if (!vmal_reg)
         return errno = ENOMEM, nullptr;
 

kernel/kernel/modules.cpp

@@ -241,7 +241,7 @@ void *module_allocate_pages(size_t size, int prot)
 {
     size_t pages = vm_size_to_pages(size);
 
-    void *p = vmalloc(pages, VM_TYPE_MODULE, prot);
+    void *p = vmalloc(pages, VM_TYPE_MODULE, prot, GFP_KERNEL);
     return p;
 }
 

kernel/kernel/perf.cpp

@@ -53,7 +53,7 @@ static int perf_probe_enable_wait()
             fg[i].nentries = FLAME_GRAPH_NENTRIES;
             fg[i].fge = (flame_graph_entry *) vmalloc(
                 vm_size_to_pages(sizeof(flame_graph_entry) * FLAME_GRAPH_NENTRIES), VM_TYPE_REGULAR,
-                VM_READ | VM_WRITE);
+                VM_READ | VM_WRITE, GFP_KERNEL);
             assert(fg[i].fge != nullptr);
         }
     }
@@ -99,7 +99,7 @@ static int perf_probe_enable()
             fg[i].nentries = FLAME_GRAPH_NENTRIES;
             fg[i].fge = (flame_graph_entry *) vmalloc(
                 vm_size_to_pages(sizeof(flame_graph_entry) * FLAME_GRAPH_NENTRIES), VM_TYPE_REGULAR,
-                VM_READ | VM_WRITE);
+                VM_READ | VM_WRITE, GFP_KERNEL);
             assert(fg[i].fge != nullptr);
         }
     }

kernel/kernel/tty/vt/vterm.cpp

@@ -12,8 +12,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <uapi/ioctls.h>
-#include <onyx/types.h>
 
 #include <onyx/dpc.h>
 #include <onyx/font.h>
@@ -24,14 +22,18 @@
 #include <onyx/input/keys.h>
 #include <onyx/input/state.h>
 #include <onyx/intrinsics.h>
+#include <onyx/page.h>
 #include <onyx/scheduler.h>
 #include <onyx/semaphore.h>
 #include <onyx/serial.h>
 #include <onyx/thread.h>
 #include <onyx/tty.h>
+#include <onyx/types.h>
 #include <onyx/utf8.h>
 #include <onyx/vm.h>
 
+#include <uapi/ioctls.h>
+
 #include <onyx/utility.hpp>
 
 struct tty;
@@ -1374,7 +1376,7 @@ void vterm_init(struct tty *tty)
     vt->fb = fb;
     vt->cells =
         (console_cell *) vmalloc(vm_size_to_pages(vt->columns * vt->rows * sizeof(*vt->cells)),
-                                 VM_TYPE_REGULAR, VM_READ | VM_WRITE);
+                                 VM_TYPE_REGULAR, VM_READ | VM_WRITE, GFP_KERNEL);
     assert(vt->cells != NULL);
 
     vt->fg = default_fg;