From 05651ccf1ece4c6afaf24de5c4d9eb572c39c135 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 9 Jun 2023 00:15:27 +0100 Subject: [PATCH] page: Add gfp_flags to memory alloc interfaces Also add GFP_KERNEL for default gfp flags. This work was derived from KTSAN work, hence why some changes don't make much sense out-of-context. Since KTSAN is not stable and will take a good while to become so, let's queue this up already, as an important patch. Signed-off-by: Pedro Falcato --- kernel/arch/riscv64/scheduler.cpp | 2 +- kernel/arch/x86_64/thread.cpp | 8 +++++--- kernel/drivers/ahci/ahci.cpp | 2 +- kernel/include/onyx/kunit.h | 2 +- kernel/include/onyx/mm/pool.hpp | 19 +++++++++++-------- kernel/include/onyx/page.h | 19 +++++++++++++++---- kernel/include/onyx/vm.h | 3 ++- kernel/kernel/compression.cpp | 2 +- kernel/kernel/kcov.cpp | 4 ++-- kernel/kernel/mm/page.cpp | 14 ++++++++++++++ kernel/kernel/mm/slab.cpp | 2 +- kernel/kernel/mm/vmalloc.cpp | 20 ++++++++++++-------- kernel/kernel/modules.cpp | 2 +- kernel/kernel/perf.cpp | 4 ++-- kernel/kernel/tty/vt/vterm.cpp | 8 +++++--- 15 files changed, 74 insertions(+), 37 deletions(-) diff --git a/kernel/arch/riscv64/scheduler.cpp b/kernel/arch/riscv64/scheduler.cpp index 620e68de5..e7daae451 100644 --- a/kernel/arch/riscv64/scheduler.cpp +++ b/kernel/arch/riscv64/scheduler.cpp @@ -128,7 +128,7 @@ thread *sched_spawn_thread(registers_t *regs, unsigned int flags, void *tp) new_thread->refcount = 1; new_thread->kernel_stack = - static_cast(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE)); + static_cast(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE, GFP_KERNEL)); if (!new_thread->kernel_stack) { diff --git a/kernel/arch/x86_64/thread.cpp b/kernel/arch/x86_64/thread.cpp index 9a2e224ce..063ae3a8b 100644 --- a/kernel/arch/x86_64/thread.cpp +++ b/kernel/arch/x86_64/thread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2021 Pedro Falcato + * Copyright (c) 2016 - 2023 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information * @@ -170,6 +170,7 @@ extern "C" void thread_finish_destruction(void *___thread) thread *sched_spawn_thread(registers_t *regs, unsigned int flags, void *fs) { thread *new_thread = new thread; + uintptr_t *thr_stack_alloc = nullptr; if (!new_thread) return NULL; @@ -211,9 +212,10 @@ thread *sched_spawn_thread(registers_t *regs, unsigned int flags, void *fs) new_thread->refcount = 1; - new_thread->kernel_stack = - static_cast(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE)); + thr_stack_alloc = + static_cast(vmalloc(pages, VM_TYPE_STACK, VM_READ | VM_WRITE, GFP_KERNEL)); + new_thread->kernel_stack = thr_stack_alloc; if (!new_thread->kernel_stack) { goto error; diff --git a/kernel/drivers/ahci/ahci.cpp b/kernel/drivers/ahci/ahci.cpp index 0f815b17a..52ef0d427 100644 --- a/kernel/drivers/ahci/ahci.cpp +++ b/kernel/drivers/ahci/ahci.cpp @@ -725,7 +725,7 @@ bool ahci_io_queue::init(ahci_hba_memory_regs_t *hba, ahci_port_t *port, ahci_po goto error; /* The fisb is 1024 bytes in size, with 1024 alignment */ - virtual_fisb = vmalloc(1, VM_TYPE_REGULAR, VM_WRITE | VM_READ); + virtual_fisb = vmalloc(1, VM_TYPE_REGULAR, VM_WRITE | VM_READ, GFP_KERNEL); if (!virtual_fisb) goto error; diff --git a/kernel/include/onyx/kunit.h b/kernel/include/onyx/kunit.h index 0ca93cc85..0b85c0dd6 100644 --- a/kernel/include/onyx/kunit.h +++ b/kernel/include/onyx/kunit.h @@ -28,7 +28,7 @@ struct test constexpr test(const char* name) : name_{name} { } - virtual void do_test(); + virtual void do_test() = 0; }; }; // namespace onx diff --git a/kernel/include/onyx/mm/pool.hpp b/kernel/include/onyx/mm/pool.hpp index 8ecd47a53..d3d1015ae 100644 --- a/kernel/include/onyx/mm/pool.hpp +++ b/kernel/include/onyx/mm/pool.hpp @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2023 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT */ #ifndef _ONYX_MM_POOL_HPP @@ -310,7 +312,7 @@ class memory_pool seg->~memory_pool_segment(); } - NO_ASAN bool expand_pool() + NO_ASAN bool expand_pool(unsigned int gfp_flags) { // std::cout << "Expanding pool.\n"; auto allocation_size = memory_pool_segment::memory_pool_size(); @@ -320,8 +322,8 @@ class memory_pool if constexpr (using_vm) { - void *vmalloc_seg = - vmalloc(allocation_size >> PAGE_SHIFT, VM_TYPE_REGULAR, VM_READ | VM_WRITE); + void *vmalloc_seg = vmalloc(allocation_size >> PAGE_SHIFT, VM_TYPE_REGULAR, + VM_READ | VM_WRITE, gfp_flags); if (!vmalloc_seg) return false; seg.set_vmalloc_seg(vmalloc_seg); @@ -329,8 +331,9 @@ class memory_pool } else { - struct page *pages = alloc_pages(allocation_size >> PAGE_SHIFT, - PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS); + struct page *pages = + alloc_pages(allocation_size >> PAGE_SHIFT, + PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS | gfp_flags); if (!pages) return false; seg.set_pages(pages); @@ -457,13 +460,13 @@ class memory_pool } NO_ASAN - T *allocate() + T *allocate(unsigned int gfp_flags = GFP_KERNEL) { scoped_lock guard{lock}; while (!free_chunk_head) { - if (!expand_pool()) + if (!expand_pool(gfp_flags)) { // std::cout << "mmap failed\n"; return nullptr; diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index b09e0ad42..5d2254012 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -158,13 +158,24 @@ void page_init(size_t memory_size, unsigned long maxpfn); unsigned int page_hash(uintptr_t p); struct page *phys_to_page(uintptr_t phys); + +/** + * @brief Retrieve the struct page from a physical address + * This may fail if phys > maxpfn + * @param phys Physical address + * @return Struct page, or NULL if > maxpfn + */ +struct page *phys_to_page_mayfail(uintptr_t phys); struct page *page_add_page(void *paddr); struct page *page_add_page_late(void *paddr); -#define PAGE_ALLOC_CONTIGUOUS (1 << 0) -#define PAGE_ALLOC_NO_ZERO (1 << 1) -#define PAGE_ALLOC_4GB_LIMIT (1 << 2) -#define PAGE_ALLOC_INTERNAL_DEBUG (1 << 3) +#define PAGE_ALLOC_CONTIGUOUS (1 << 0) +#define PAGE_ALLOC_NO_ZERO (1 << 1) +#define PAGE_ALLOC_4GB_LIMIT (1 << 2) +#define PAGE_ALLOC_INTERNAL_DEBUG (1 << 3) +#define PAGE_ALLOC_NO_SANITIZER_SHADOW (1 << 4) + +#define GFP_KERNEL 0 static inline bool __page_should_zero(unsigned long flags) { diff --git a/kernel/include/onyx/vm.h b/kernel/include/onyx/vm.h index 6ac288081..dfa08dcb8 100644 --- a/kernel/include/onyx/vm.h +++ b/kernel/include/onyx/vm.h @@ -351,9 +351,10 @@ void *vm_get_fallback_pgd(); * @param pages The number of pages. * @param type The type of allocation. * @param perms The permissions on the allocation. + * @param gfp_flags GFP flags * @return A pointer to the new allocation, or NULL with errno set on failure. */ -void *vmalloc(size_t pages, int type, int perms); +void *vmalloc(size_t pages, int type, int perms, unsigned int gfp_flags); /** * @brief Frees a region of memory previously allocated by vmalloc. diff --git a/kernel/kernel/compression.cpp b/kernel/kernel/compression.cpp index 83f7ceb9f..97c019028 100644 --- a/kernel/kernel/compression.cpp +++ b/kernel/kernel/compression.cpp @@ -60,7 +60,7 @@ expected, int> create_decompression_stream( bool decompress_bytestream::init(size_t len) { - buf = vmalloc(vm_size_to_pages(len), VM_TYPE_REGULAR, VM_WRITE | VM_READ); + buf = vmalloc(vm_size_to_pages(len), VM_TYPE_REGULAR, VM_WRITE | VM_READ, GFP_KERNEL); if (!buf) return false; this->len = len; diff --git a/kernel/kernel/kcov.cpp b/kernel/kernel/kcov.cpp index 0b1344933..a9ba86715 100644 --- a/kernel/kernel/kcov.cpp +++ b/kernel/kernel/kcov.cpp @@ -133,8 +133,8 @@ int kcov_init_trace(unsigned long nr_elems, struct file *f) return -EINVAL; // We use a vmalloc region and set up a VMO for it. This VMO is then mmap'd. - auto buffer = - (unsigned long *) vmalloc(vm_size_to_pages(size), VM_TYPE_REGULAR, VM_READ | VM_WRITE); + auto buffer = (unsigned long *) vmalloc(vm_size_to_pages(size), VM_TYPE_REGULAR, + VM_READ | VM_WRITE, GFP_KERNEL); if (!buffer) return -ENOMEM; diff --git a/kernel/kernel/mm/page.cpp b/kernel/kernel/mm/page.cpp index d90772a5a..86ce4daaf 100644 --- a/kernel/kernel/mm/page.cpp +++ b/kernel/kernel/mm/page.cpp @@ -51,6 +51,20 @@ struct page *phys_to_page(uintptr_t phys) return page_map + pfn - base_pfn; } +/** + * @brief Retrieve the struct page from a physical address + * This may fail if phys > maxpfn + * @param phys Physical address + * @return Struct page, or NULL if > maxpfn + */ +struct page *phys_to_page_mayfail(uintptr_t phys) +{ + unsigned long pfn = phys >> PAGE_SHIFT; + if (pfn > maxpfn) + return nullptr; + return page_map + pfn - base_pfn; +} + extern unsigned char kernel_start; extern unsigned char kernel_end; uint64_t kernel_phys_offset = 0; diff --git a/kernel/kernel/mm/slab.cpp b/kernel/kernel/mm/slab.cpp index 40f54d070..9982def5f 100644 --- a/kernel/kernel/mm/slab.cpp +++ b/kernel/kernel/mm/slab.cpp @@ -397,7 +397,7 @@ NO_ASAN static struct slab *kmem_cache_create_slab(struct slab_cache *cache, uns } else { - start = (char *) vmalloc(slab_size >> PAGE_SHIFT, VM_TYPE_HEAP, VM_READ | VM_WRITE); + start = (char *) vmalloc(slab_size >> PAGE_SHIFT, VM_TYPE_HEAP, VM_READ | VM_WRITE, flags); if (!start) return nullptr; } diff --git a/kernel/kernel/mm/vmalloc.cpp b/kernel/kernel/mm/vmalloc.cpp index 89466fbbe..e624e1f73 100644 --- a/kernel/kernel/mm/vmalloc.cpp +++ b/kernel/kernel/mm/vmalloc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Pedro Falcato + * Copyright (c) 2022 - 2023 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information * @@ -131,12 +131,13 @@ static memory_pool pool; * @param start Start of the region * @param pages Number of pages of the region * @param perms Permissions + * @param gfp_flags GFP flags * @return Pointer to a vmalloc_region, or nullptr */ -struct vmalloc_region *vmalloc_insert_region(struct vmalloc_tree *tree, unsigned long start, - size_t pages, int perms) +static struct vmalloc_region *vmalloc_insert_region(struct vmalloc_tree *tree, unsigned long start, + size_t pages, int perms, unsigned int gfp_flags) { - auto reg = pool.allocate(); + auto reg = pool.allocate(gfp_flags); if (!reg) return nullptr; reg->addr = start; @@ -172,9 +173,10 @@ struct vmalloc_region *vmalloc_insert_region(struct vmalloc_tree *tree, unsigned * @param pages The number of pages. * @param type The type of allocation. * @param perms The permissions on the allocation. + * @param gfp_flags GFP flags * @return A pointer to the new allocation, or NULL with errno set on failure. */ -void *vmalloc(size_t pages, int type, int perms) +void *vmalloc(size_t pages, int type, int perms, unsigned int gfp_flags) { scoped_lock g{vmalloc_tree.lock}; auto start = vmalloc_allocate_base(&vmalloc_tree, 0, pages << PAGE_SHIFT); @@ -182,7 +184,8 @@ void *vmalloc(size_t pages, int type, int perms) if (start + (pages << PAGE_SHIFT) > vmalloc_tree.start + vmalloc_tree.length) return errno = ENOMEM, nullptr; - auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, perms); + auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, perms, + gfp_flags | PAGE_ALLOC_NO_SANITIZER_SHADOW); if (!vmal_reg) return errno = ENOMEM, nullptr; @@ -199,7 +202,7 @@ void *vmalloc(size_t pages, int type, int perms) } #endif - auto pgs = alloc_pages(pages, 0); + auto pgs = alloc_pages(pages, gfp_flags); if (!pgs) { delvmr(); @@ -336,7 +339,8 @@ void *mmiomap(void *phys, size_t size, size_t flags) if (start + (pages << PAGE_SHIFT) > vmalloc_tree.start + vmalloc_tree.length) return errno = ENOMEM, nullptr; - auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, flags); + auto vmal_reg = vmalloc_insert_region(&vmalloc_tree, start, pages, flags, + GFP_KERNEL | PAGE_ALLOC_NO_SANITIZER_SHADOW); if (!vmal_reg) return errno = ENOMEM, nullptr; diff --git a/kernel/kernel/modules.cpp b/kernel/kernel/modules.cpp index fc31bbaae..74cdc2408 100644 --- a/kernel/kernel/modules.cpp +++ b/kernel/kernel/modules.cpp @@ -241,7 +241,7 @@ void *module_allocate_pages(size_t size, int prot) { size_t pages = vm_size_to_pages(size); - void *p = vmalloc(pages, VM_TYPE_MODULE, prot); + void *p = vmalloc(pages, VM_TYPE_MODULE, prot, GFP_KERNEL); return p; } diff --git a/kernel/kernel/perf.cpp b/kernel/kernel/perf.cpp index c30a599ef..a454f4e27 100644 --- a/kernel/kernel/perf.cpp +++ b/kernel/kernel/perf.cpp @@ -53,7 +53,7 @@ static int perf_probe_enable_wait() fg[i].nentries = FLAME_GRAPH_NENTRIES; fg[i].fge = (flame_graph_entry *) vmalloc( vm_size_to_pages(sizeof(flame_graph_entry) * FLAME_GRAPH_NENTRIES), VM_TYPE_REGULAR, - VM_READ | VM_WRITE); + VM_READ | VM_WRITE, GFP_KERNEL); assert(fg[i].fge != nullptr); } } @@ -99,7 +99,7 @@ static int perf_probe_enable() fg[i].nentries = FLAME_GRAPH_NENTRIES; fg[i].fge = (flame_graph_entry *) vmalloc( vm_size_to_pages(sizeof(flame_graph_entry) * FLAME_GRAPH_NENTRIES), VM_TYPE_REGULAR, - VM_READ | VM_WRITE); + VM_READ | VM_WRITE, GFP_KERNEL); assert(fg[i].fge != nullptr); } } diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index 48afce4a6..c03263356 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -12,8 +12,6 @@ #include #include #include -#include -#include #include #include @@ -24,14 +22,18 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include +#include + #include struct tty; @@ -1374,7 +1376,7 @@ void vterm_init(struct tty *tty) vt->fb = fb; vt->cells = (console_cell *) vmalloc(vm_size_to_pages(vt->columns * vt->rows * sizeof(*vt->cells)), - VM_TYPE_REGULAR, VM_READ | VM_WRITE); + VM_TYPE_REGULAR, VM_READ | VM_WRITE, GFP_KERNEL); assert(vt->cells != NULL); vt->fg = default_fg;