diff --git a/Cargo.lock b/Cargo.lock index 35c2f6ea2c..01d3aeb62b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3718,6 +3718,14 @@ dependencies = [ "sync_irq", ] +[[package]] +name = "test_1gb_huge_pages" +version = "0.1.0" +dependencies = [ + "app_io", + "memory", +] + [[package]] name = "test_aligned_page_allocation" version = "0.1.0" @@ -3785,6 +3793,16 @@ dependencies = [ "root", ] +[[package]] +name = "test_huge_pages" +version = "0.1.0" +dependencies = [ + "app_io", + "frame_allocator", + "memory", + "page_allocator", +] + [[package]] name = "test_identity_mapping" version = "0.1.0" diff --git a/applications/test_1gb_huge_pages/Cargo.toml b/applications/test_1gb_huge_pages/Cargo.toml new file mode 100644 index 0000000000..49ffde7bdc --- /dev/null +++ b/applications/test_1gb_huge_pages/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "test_1gb_huge_pages" +version = "0.1.0" +authors = ["Noah Mogensen "] +description = "Application for testing allocation, mapping, and memory access for 1gb huge pages." + +[dependencies] + +[dependencies.memory] +path = "../../kernel/memory" + +[dependencies.app_io] +path = "../../kernel/app_io" + +# [dependencies.application_main_fn] +# path = "../../compiler_plugins" diff --git a/applications/test_1gb_huge_pages/src/lib.rs b/applications/test_1gb_huge_pages/src/lib.rs new file mode 100644 index 0000000000..728031a709 --- /dev/null +++ b/applications/test_1gb_huge_pages/src/lib.rs @@ -0,0 +1,62 @@ +#![no_std] +#[macro_use] extern crate alloc; +#[macro_use] extern crate app_io; + +extern crate memory; + +use alloc::vec::Vec; +use alloc::string::String; +use memory::{ + PteFlags, + VirtualAddress, PhysicalAddress, + allocate_frames_at, allocate_pages_at +}; + +pub fn main(_args: Vec) -> isize { + println!("NOTE: Before running, make sure you Theseus has been run with enough memory (make orun QEMU_MEMORY=5G)."); + let kernel_mmi_ref = memory::get_kernel_mmi_ref() + .ok_or("KERNEL_MMI was not yet initialized!") + .unwrap(); + + // Now the same for 1gb pages + let mut aligned_1gb_page = allocate_pages_at( + VirtualAddress::new_canonical(0x40000000), + 512*512).expect("Failed to allocate range for 1GiB page. Make sure you have enough memory for the kernel (compile with make orun QEMU_MEMORY=3G)."); + aligned_1gb_page.to_1gb_allocated_pages(); + + let aligned_4k_frames = allocate_frames_at( + PhysicalAddress::new_canonical(0x40000000), //0x1081A000 + 512 * 512).expect("Failed to allocate enough frames at desired address. Make sure you have enough memory for the kernel (compile with make orun QEMU_MEMORY=3G)."); + + let mut mapped_1gb_page = kernel_mmi_ref.lock().page_table.map_allocated_pages_to( + aligned_1gb_page, + aligned_4k_frames, + PteFlags::new().valid(true).writable(true), + ).expect("test_huge_pages: call to map_allocated_pages failed for 1GiB page"); + + kernel_mmi_ref.lock().page_table.dump_pte(mapped_1gb_page.start_address()); + + // See if address can be translated + let translated = kernel_mmi_ref.lock() + .page_table.translate(VirtualAddress::new_canonical(0x40000000)) + .unwrap(); + println!("Virtual address 0x40000000-> {}", translated); + + let val = mapped_1gb_page + .as_type_mut::(0) + .expect("test huge pages: call to as_type_mut() on mapped 2Mb page failed"); + println!("Value at offset of 1GiB huge page before updating is {}", *val); + + *val = 35; + + let updated_val = mapped_1gb_page + .as_type::(0) + .expect("test huge pages: call to as_type() on mapped 2Mb page failed"); + println!("Value at offset of 1GiB huge page after updating is {}", *updated_val); + + // Dump entries to see if dropping has unmapped everything properly + drop(mapped_1gb_page); + kernel_mmi_ref.lock().page_table.dump_pte(VirtualAddress::new_canonical(0x40000000)); + + 0 +} diff --git a/applications/test_huge_pages/Cargo.toml b/applications/test_huge_pages/Cargo.toml new file mode 100644 index 0000000000..77ebd7db4c --- /dev/null +++ b/applications/test_huge_pages/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "test_huge_pages" +version = "0.1.0" +authors = ["Noah Mogensen "] +description = "Application for testing allocation, mapping, and memory access for 2mb huge pages." + +[dependencies] + +[dependencies.memory] +path = "../../kernel/memory" + +[dependencies.frame_allocator] +path = "../../kernel/frame_allocator" + +[dependencies.page_allocator] +path = "../../kernel/page_allocator" + +[dependencies.app_io] +path = "../../kernel/app_io" + +# [dependencies.application_main_fn] +# path = "../../compiler_plugins" diff --git a/applications/test_huge_pages/src/lib.rs b/applications/test_huge_pages/src/lib.rs new file mode 100644 index 0000000000..2bceba3f32 --- /dev/null +++ b/applications/test_huge_pages/src/lib.rs @@ -0,0 +1,96 @@ +#![no_std] +#[macro_use] extern crate alloc; +#[macro_use] extern crate app_io; + +extern crate memory; +extern crate frame_allocator; +extern crate page_allocator; + +use alloc::vec::Vec; +use alloc::string::String; +use memory::{ + Page, PteFlags, + VirtualAddress, PhysicalAddress, + Page1G, Page2M, + allocate_frames_at, allocate_pages_at +}; +// For checking valid test addresses, if necessary +// use frame_allocator::dump_frame_allocator_state; +// use page_allocator::dump_page_allocator_state; + +pub fn main(_args: Vec) -> isize { + println!("NOTE: Before running, make sure you Theseus has been run with enough memory (make orun QEMU_MEMORY=3G)."); + + let kernel_mmi_ref = memory::get_kernel_mmi_ref() + .ok_or("KERNEL_MMI was not yet initialized!") + .unwrap(); + + // Preliminary tests + let page1g = Page::::containing_address_1gb(VirtualAddress::zero()); + println!("Size of page 1 is {:?}", page1g.page_size()); + let page2m = Page::::containing_address_2mb(VirtualAddress::zero()); + println!("Size of page 2 is {:?}", page2m.page_size()); + + + // match page1g.page_size() { + // PAGE_1GB_SIZE => println!("Page 1 recognized as 1GiB"), + // _ => println!("Page 1 not recognized as 1GiB"), + // } + // match page2m.page_size() { + // PAGE_2MB_SIZE => println!("Page 2 recognized as 2MiB"), + // _ => println!("Page 2 not recognized as 2MiB"), + // } + // let _allocated_1g = allocate_1gb_pages(1) + // .ok_or("test_huge_pages: could not allocate 1GiB page") + // .unwrap(); + // let _allocated_2m = allocate_2mb_pages(1) + // .ok_or("test_huge_pages: could not allocate 2MiB page") + // .unwrap(); + // println!("Huge pages successfully allocated!"); + // // end preliminary tests + + let mut aligned_2mb_page = allocate_pages_at( + VirtualAddress::new_canonical(0x60000000), + 512).expect("Could not allocate pages. Make sure you have enough memory for the kernel (compile with make orun QEMU_MEMORY=3G)."); + aligned_2mb_page.to_2mb_allocated_pages(); + + // frame allocator has not been modified to deal with huge frames yet + let aligned_4k_frames = allocate_frames_at( + PhysicalAddress::new_canonical(0x60000000), + 512).expect("Could not allocate frames. Make sure you have enough memory for the kernel (compile with make orun QEMU_MEMORY=3G)."); + + let allocated_2mb_frames = aligned_4k_frames.into_2mb_allocated_frames().expect("Could not convert range of allocated frames into huge allocated frames"); + + let mut mapped_2mb_page = kernel_mmi_ref.lock().page_table.map_allocated_pages_to( + aligned_2mb_page, + allocated_2mb_frames, + PteFlags::new().valid(true).writable(true), + ).expect("test_huge_pages: call to map_allocated_pages failed"); + + kernel_mmi_ref.lock().page_table.dump_pte(mapped_2mb_page.start_address()); + + // See if address can be translated + let translated = kernel_mmi_ref.lock() + .page_table.translate(mapped_2mb_page.start_address()) + .unwrap(); + println!("Virtual address {} -> {}", mapped_2mb_page.start_address(), translated); + + + // Testing mem access with huge pages + let val = mapped_2mb_page + .as_type_mut::(0) + .expect("test huge pages: call to as_type_mut() on mapped 2Mb page failed"); + println!("Value at offset of 2Mib huge page before updating is {}", *val); + + *val = 35; + + let updated_val = mapped_2mb_page + .as_type::(0) + .expect("test huge pages: call to as_type() on mapped 2Mb page failed"); + println!("Value at offset of 2Mib huge page after updating is {}", *updated_val); + + // Dump entries to see if dropping has unmapped everything properly + drop(mapped_2mb_page); + kernel_mmi_ref.lock().page_table.dump_pte(VirtualAddress::new_canonical(0x20000000)); + 0 +} diff --git a/kernel/frame_allocator/src/lib.rs b/kernel/frame_allocator/src/lib.rs index a9c09425d7..93a181ad3f 100644 --- a/kernel/frame_allocator/src/lib.rs +++ b/kernel/frame_allocator/src/lib.rs @@ -1,17 +1,17 @@ //! Provides an allocator for physical memory frames. -//! The minimum unit of allocation is a single frame. +//! The minimum unit of allocation is a single frame. //! //! This is currently a modified and more complex version of the `page_allocator` crate. //! TODO: extract the common code and create a generic allocator that can be specialized to allocate pages or frames. -//! -//! This also supports early allocation of frames before heap allocation is available, -//! and does so behind the scenes using the same single interface. +//! +//! This also supports early allocation of frames before heap allocation is available, +//! and does so behind the scenes using the same single interface. //! Early pre-heap allocations are limited to tracking a small number of available chunks (currently 32). -//! +//! //! Once heap allocation is available, it uses a dynamically-allocated list of frame chunks to track allocations. -//! -//! The core allocation function is [`allocate_frames_deferred()`](fn.allocate_frames_deferred.html), -//! but there are several convenience functions that offer simpler interfaces for general usage. +//! +//! The core allocation function is [`allocate_frames_deferred()`](fn.allocate_frames_deferred.html), +//! but there are several convenience functions that offer simpler interfaces for general usage. //! //! # Notes and Missing Features //! This allocator only makes one attempt to merge deallocated frames into existing @@ -23,6 +23,7 @@ #![allow(clippy::blocks_in_if_conditions)] #![allow(incomplete_features)] #![feature(adt_const_params)] +#![feature(step_trait)] extern crate alloc; #[cfg(test)] @@ -31,6 +32,7 @@ mod test; mod static_array_rb_tree; // mod static_array_linked_list; + use core::{borrow::Borrow, cmp::{Ordering, min, max}, fmt, mem, ops::{Deref, DerefMut}}; use intrusive_collections::Bound; use kernel_config::memory::*; @@ -42,40 +44,39 @@ use static_assertions::assert_not_impl_any; const FRAME_4K_SIZE_IN_BYTES: usize = PAGE_SIZE; -// Note: we keep separate lists for "free, general-purpose" areas and "reserved" areas, as it's much faster. +// Note: we keep separate lists for "free, general-purpose" areas and "reserved" areas, as it's much faster. -/// The single, system-wide list of free physical memory frames available for general usage. -static FREE_GENERAL_FRAMES_LIST: Mutex> = Mutex::new(StaticArrayRBTree::empty()); -/// The single, system-wide list of free physical memory frames reserved for specific usage. -static FREE_RESERVED_FRAMES_LIST: Mutex> = Mutex::new(StaticArrayRBTree::empty()); +/// The single, system-wide list of free physical memory frames available for general usage. +static FREE_GENERAL_FRAMES_LIST: Mutex> = Mutex::new(StaticArrayRBTree::empty()); +/// The single, system-wide list of free physical memory frames reserved for specific usage. +static FREE_RESERVED_FRAMES_LIST: Mutex> = Mutex::new(StaticArrayRBTree::empty()); /// The fixed list of all known regions that are available for general use. -/// This does not indicate whether these regions are currently allocated, +/// This does not indicate whether these regions are currently allocated, /// rather just where they exist and which regions are known to this allocator. static GENERAL_REGIONS: Mutex> = Mutex::new(StaticArrayRBTree::empty()); -/// The fixed list of all known regions that are reserved for specific purposes. -/// This does not indicate whether these regions are currently allocated, +/// The fixed list of all known regions that are reserved for specific purposes. +/// This does not indicate whether these regions are currently allocated, /// rather just where they exist and which regions are known to this allocator. static RESERVED_REGIONS: Mutex> = Mutex::new(StaticArrayRBTree::empty()); - /// Initialize the frame allocator with the given list of available and reserved physical memory regions. /// /// Any regions in either of the lists may overlap, this is checked for and handled properly. /// Reserved regions take priority -- if a reserved region partially or fully overlaps any part of a free region, -/// that portion will be considered reserved, not free. -/// -/// The iterator (`R`) over reserved physical memory regions must be cloneable, -/// as this runs before heap allocation is available, and we may need to iterate over it multiple times. -/// +/// that portion will be considered reserved, not free. +/// +/// The iterator (`R`) over reserved physical memory regions must be cloneable, +/// as this runs before heap allocation is available, and we may need to iterate over it multiple times. +/// /// ## Return /// Upon success, this function returns a callback function that allows the caller -/// (the memory subsystem init function) to convert a range of unmapped frames +/// (the memory subsystem init function) to convert a range of unmapped frames /// back into an [`UnmappedFrames`] object. pub fn init( free_physical_memory_areas: F, reserved_physical_memory_areas: R, -) -> Result UnmappedFrames, &'static str> +) -> Result UnmappedFrames, &'static str> where P: Borrow, F: IntoIterator, R: IntoIterator + Clone, @@ -83,7 +84,7 @@ pub fn init( if FREE_GENERAL_FRAMES_LIST .lock().len() != 0 || FREE_RESERVED_FRAMES_LIST.lock().len() != 0 || GENERAL_REGIONS .lock().len() != 0 || - RESERVED_REGIONS .lock().len() != 0 + RESERVED_REGIONS .lock().len() != 0 { return Err("BUG: Frame allocator was already initialized, cannot be initialized twice."); } @@ -142,7 +143,7 @@ pub fn init( } - // Finally, one last sanity check -- ensure no two regions overlap. + // Finally, one last sanity check -- ensure no two regions overlap. let all_areas = free_list[..free_list_idx].iter().flatten() .chain(reserved_list.iter().flatten()); for (i, elem) in all_areas.clone().enumerate() { @@ -181,7 +182,7 @@ pub fn init( } -/// The main logic of the initialization routine +/// The main logic of the initialization routine /// used to populate the list of free frame chunks. /// /// This function recursively iterates over the given `area` of frames @@ -196,9 +197,9 @@ fn check_and_add_free_region( where P: Borrow, R: IntoIterator + Clone, { - // This will be set to the frame that is the start of the current free region. + // This will be set to the frame that is the start of the current free region. let mut current_start = *area.start(); - // This will be set to the frame that is the end of the current free region. + // This will be set to the frame that is the end of the current free region. let mut current_end = *area.end(); // trace!("looking at sub-area {:X?} to {:X?}", current_start, current_end); @@ -268,7 +269,7 @@ impl PhysicalMemoryRegion { PhysicalMemoryRegion { frames, typ } } - /// Returns a new `PhysicalMemoryRegion` with an empty range of frames. + /// Returns a new `PhysicalMemoryRegion` with an empty range of frames. #[allow(unused)] const fn empty() -> PhysicalMemoryRegion { PhysicalMemoryRegion { @@ -309,8 +310,8 @@ impl Borrow for &'_ PhysicalMemoryRegion { pub enum MemoryRegionType { /// Memory that is available for any general purpose. Free, - /// Memory that is reserved for special use and is only ever allocated from if specifically requested. - /// This includes custom memory regions added by third parties, e.g., + /// Memory that is reserved for special use and is only ever allocated from if specifically requested. + /// This includes custom memory regions added by third parties, e.g., /// device memory discovered and added by device drivers later during runtime. Reserved, /// Memory of an unknown type. @@ -323,7 +324,7 @@ pub enum MemoryRegionType { /// /// Each `Frames` object is globally unique, meaning that the owner of a `Frames` object /// has globally-exclusive access to the range of frames it contains. -/// +/// /// A `Frames` object can be in one of four states: /// * `Free`: frames are owned by the frame allocator and have not been allocated for any use. /// * `Allocated`: frames have been removed from the allocator's free list and are owned elsewhere; @@ -343,7 +344,7 @@ pub enum MemoryRegionType { /// ``` /// (Free) <---> (Allocated) --> (Mapped) --> (Unmapped) --> (Allocated) <---> (Free) /// ``` -/// +/// /// # Ordering and Equality /// /// `Frames` implements the `Ord` trait, and its total ordering is ONLY based on @@ -353,7 +354,7 @@ pub enum MemoryRegionType { /// both of which are also based ONLY on the **starting** `Frame` of the `Frames`. /// Thus, comparing two `Frames` with the `==` or `!=` operators may not work as expected. /// since it ignores their actual range of frames. -/// +/// /// Similarly, `Frames` implements the `Borrow` trait to return a `Frame`, /// not a `FrameRange`. This is required so we can search for `Frames` in a sorted collection /// using a `Frame` value. @@ -436,6 +437,28 @@ impl AllocatedFrames

{ _phantom: core::marker::PhantomData, } } + + pub fn into_2mb_allocated_frames(self) -> Result { + Ok(AllocatedFrames { + typ: self.typ, + frames: FrameRangeSized::Huge2MiB( + FrameRange::::try_from(self.frames + .range() + .unwrap() + .clone())?) + }) + } + + pub fn into_1gb_allocated_frames(self) -> Result { + Ok(AllocatedFrames { + typ: self.typ, + frames: FrameRangeSized::Huge1GiB( + FrameRange::::try_from(self.frames + .range() + .unwrap() + .clone())?) + }) + } } impl UnmappedFrames { @@ -455,7 +478,7 @@ impl UnmappedFrames { /// This function is a callback used to convert `UnmappedFrameRange` into `UnmappedFrames`. /// /// `UnmappedFrames` represents frames that have been unmapped by a page that had -/// previously exclusively mapped them, indicating that no others pages have been mapped +/// previously exclusively mapped them, indicating that no others pages have been mapped /// to those same frames, and thus, those frames can be safely deallocated. /// /// This exists to break the cyclic dependency chain between this crate and @@ -488,8 +511,8 @@ impl Drop for Frames { FREE_RESERVED_FRAMES_LIST.lock() } else { FREE_GENERAL_FRAMES_LIST.lock() - }; - + }; + match &mut list.0 { // For early allocations, just add the deallocated chunk to the free pages list. Inner::Array(_) => { @@ -499,8 +522,8 @@ impl Drop for Frames { error!("Failed to insert deallocated frames into the list (array). The initial static array should be created with a larger size."); } } - - // For full-fledged deallocations, determine if we can merge the deallocated frames + + // For full-fledged deallocations, determine if we can merge the deallocated frames // with an existing contiguously-adjacent chunk or if we need to insert a new chunk. Inner::RBTree(ref mut tree) => { let mut cursor_mut = tree.lower_bound_mut(Bound::Included(free_frames.start())); @@ -586,6 +609,10 @@ impl<'f, P: PageSize> IntoIterator for &'f AllocatedFrames

{ _owner: self, range: self.frame_range.iter(), } + // AllocatedFramesIter { + // _owner: self, + // range: self.range().unwrap().iter(), + // } } } @@ -606,6 +633,11 @@ pub struct AllocatedFramesIter<'f, P: PageSize> { impl<'f, P: PageSize> Iterator for AllocatedFramesIter<'f, P> { type Item = AllocatedFrame<'f, P>; fn next(&mut self) -> Option { + // match self._owner.page_size() { + // MemChunkSize::Normal4K => {} + // MemChunkSize::Huge2M => {} + // MemChunkSize::Huge1G => {} + // } self.range.next().map(|frame| AllocatedFrame { frame, _phantom: core::marker::PhantomData, @@ -614,8 +646,9 @@ impl<'f, P: PageSize> Iterator for AllocatedFramesIter<'f, P> { } } + /// A reference to a single frame within a range of `AllocatedFrames`. -/// +/// /// The lifetime of this type is tied to the lifetime of its owning `AllocatedFrames`. #[derive(Debug)] pub struct AllocatedFrame<'f, P: PageSize> { @@ -644,7 +677,7 @@ impl Frames { self.typ } - /// Returns a new `Frames` with an empty range of frames. + /// Returns a new `Frames` with an empty range of frames. /// Can be used as a placeholder, but will not permit any real usage. pub const fn empty() -> Frames { Frames { @@ -658,19 +691,19 @@ impl Frames { /// This function performs no allocation or re-mapping, it exists for convenience and usability purposes. /// /// The given `other` must be physically contiguous with `self`, i.e., come immediately before or after `self`. - /// That is, either `self.start == other.end + 1` or `self.end + 1 == other.start` must be true. + /// That is, either `self.start == other.end + 1` or `self.end + 1 == other.start` must be true. /// /// If either of those conditions are met, `self` is modified and `Ok(())` is returned, /// otherwise `Err(other)` is returned. pub fn merge(&mut self, other: Self) -> Result<(), Self> { - if self.is_empty() || other.is_empty() { + if self.range().unwrap().is_empty() || other.range().unwrap().is_empty() { return Err(other); } let frames = if *self.start() == *other.end() + 1 { // `other` comes contiguously before `self` FrameRange::new(*other.start(), *self.end()) - } + } else if *self.end() + 1 == *other.start() { // `self` comes contiguously before `other` FrameRange::new(*self.start(), *other.end()) @@ -687,12 +720,12 @@ impl Frames { } /// Splits up the given `Frames` into multiple smaller `Frames`. - /// + /// /// Returns a `SplitFrames` instance containing three `Frames`: /// 1. The range of frames in `self` that are before the beginning of `frames_to_extract`. /// 2. The `Frames` containing the requested range of frames, `frames_to_extract`. /// 3. The range of frames in `self` that are after the end of `frames_to_extract`. - /// + /// /// If `frames_to_extract` is not contained within `self`, then `self` is returned unchanged within an `Err`. pub fn split_range( self, @@ -702,7 +735,7 @@ impl Frames { if !self.contains_range(&frames_to_extract) { return Err(self); } - + let start_frame = *frames_to_extract.start(); let start_to_end = frames_to_extract; @@ -731,14 +764,14 @@ impl Frames { /// Splits this `Frames` into two separate `Frames` objects: /// * `[beginning : at_frame - 1]` /// * `[at_frame : end]` - /// + /// /// This function follows the behavior of [`core::slice::split_at()`], - /// thus, either one of the returned `Frames` objects may be empty. + /// thus, either one of the returned `Frames` objects may be empty. /// * If `at_frame == self.start`, the first returned `Frames` object will be empty. /// * If `at_frame == self.end + 1`, the second returned `Frames` object will be empty. - /// + /// /// Returns an `Err` containing this `Frames` if `at_frame` is otherwise out of bounds, or if `self` was empty. - /// + /// /// [`core::slice::split_at()`]: https://doc.rust-lang.org/core/primitive.slice.html#method.split_at pub fn split_at(self, at_frame: Frame

) -> Result<(Self, Self), Self> { if self.is_empty() { return Err(self); } @@ -749,7 +782,7 @@ impl Frames { let first = FrameRange::

::empty(); let second = FrameRange::

::new(at_frame, *self.end()); (first, second) - } + } else if at_frame == (*self.end() + 1) && end_of_first >= *self.start() { let first = FrameRange::

::new(*self.start(), *self.end()); let second = FrameRange::

::empty(); @@ -808,17 +841,17 @@ impl fmt::Debug for Frames { /// A series of pending actions related to frame allocator bookkeeping, -/// which may result in heap allocation. -/// -/// The actions are triggered upon dropping this struct. -/// This struct can be returned from the `allocate_frames()` family of functions -/// in order to allow the caller to precisely control when those actions -/// that may result in heap allocation should occur. -/// Such actions include adding chunks to lists of free frames or frames in use. -/// -/// The vast majority of use cases don't care about such precise control, +/// which may result in heap allocation. +/// +/// The actions are triggered upon dropping this struct. +/// This struct can be returned from the `allocate_frames()` family of functions +/// in order to allow the caller to precisely control when those actions +/// that may result in heap allocation should occur. +/// Such actions include adding chunks to lists of free frames or frames in use. +/// +/// The vast majority of use cases don't care about such precise control, /// so you can simply drop this struct at any time or ignore it -/// with a `let _ = ...` binding to instantly drop it. +/// with a `let _ = ...` binding to instantly drop it. pub struct DeferredAllocAction<'list> { /// A reference to the list into which we will insert the free general-purpose `Chunk`s. free_list: &'list Mutex>, @@ -830,7 +863,7 @@ pub struct DeferredAllocAction<'list> { free2: FreeFrames, } impl<'list> DeferredAllocAction<'list> { - fn new(free1: F1, free2: F2) -> DeferredAllocAction<'list> + fn new(free1: F1, free2: F2) -> DeferredAllocAction<'list> where F1: Into>, F2: Into>, { @@ -923,7 +956,7 @@ fn find_specific_chunk( return allocate_from_chosen_chunk(FrameRange::new(requested_frame, requested_frame + num_frames - 1), ValueRefMut::RBTree(cursor_mut), None); } else { // We found the chunk containing the requested address, but it was too small to cover all of the requested frames. - // Let's try to merge the next-highest contiguous chunk to see if those two chunks together + // Let's try to merge the next-highest contiguous chunk to see if those two chunks together // cover enough frames to fulfill the allocation request. // // trace!("Frame allocator: found chunk containing requested address, but it was too small. \ @@ -935,7 +968,7 @@ fn find_specific_chunk( cursor_mut.move_next();// cursor now points to the next chunk if let Some(next_chunk) = cursor_mut.get().map(|w| w.deref()) { if *chunk.end() + 1 == *next_chunk.start() { - // Here: next chunk was contiguous with the original chunk. + // Here: next chunk was contiguous with the original chunk. if requested_end_frame <= *next_chunk.end() { // trace!("Frame allocator: found suitably-large contiguous next {:?} after initial too-small {:?}", next_chunk, chunk); let next = cursor_mut.remove().map(|f| f.into_inner()); @@ -961,10 +994,10 @@ fn find_specific_chunk( } }; if let Some(next_chunk) = next_contiguous_chunk { - // We found a suitable chunk that came contiguously after the initial too-small chunk. + // We found a suitable chunk that came contiguously after the initial too-small chunk. // We would like to merge it into the initial chunk with just the reference (since we have a cursor pointing to it already), // but we can't get a mutable reference to the element the cursor is pointing to. - // So both chunks will be removed and then merged. + // So both chunks will be removed and then merged. return allocate_from_chosen_chunk(FrameRange::new(requested_frame, requested_frame + num_frames - 1), ValueRefMut::RBTree(cursor_mut), Some(next_chunk)); } } @@ -990,7 +1023,7 @@ fn find_any_chunk( // Skip chunks that are too-small or in the designated regions. if chunk.size_in_frames() < num_frames || chunk.typ() != MemoryRegionType::Free { continue; - } + } else { return allocate_from_chosen_chunk(FrameRange::new(*chunk.start(), *chunk.start() + num_frames - 1), ValueRefMut::Array(elem), None); } @@ -998,8 +1031,8 @@ fn find_any_chunk( } } Inner::RBTree(ref mut tree) => { - // Because we allocate new frames by peeling them off from the beginning part of a chunk, - // it's MUCH faster to start the search for free frames from higher addresses moving down. + // Because we allocate new frames by peeling them off from the beginning part of a chunk, + // it's MUCH faster to start the search for free frames from higher addresses moving down. // This results in an O(1) allocation time in the general case, until all address ranges are already in use. let mut cursor = tree.upper_bound_mut(Bound::<&FreeFrames>::Unbounded); while let Some(chunk) = cursor.get().map(|w| w.deref()) { @@ -1023,12 +1056,12 @@ fn find_any_chunk( } -/// Removes a `Frames` object from the RBTree. +/// Removes a `Frames` object from the RBTree. /// `frames_ref` is basically a wrapper over the cursor which stores the position of the frames. fn retrieve_frames_from_ref(mut frames_ref: ValueRefMut) -> Option { // Remove the chosen chunk from the free frame list. let removed_val = frames_ref.remove(); - + match removed_val { RemovedValue::Array(c) => c, RemovedValue::RBTree(option_frames) => { @@ -1037,10 +1070,10 @@ fn retrieve_frames_from_ref(mut frames_ref: ValueRefMut) -> Option, @@ -1050,7 +1083,7 @@ fn allocate_from_chosen_chunk( // Remove the initial chunk from the free frame list. let mut chosen_chunk = retrieve_frames_from_ref(initial_chunk_ref) .expect("BUG: Failed to retrieve chunk from free list"); - + // This should always succeed, since we've already checked the conditions for a merge and split. // We should return the chunks back to the list, but a failure at this point implies a bug in the frame allocator. @@ -1105,13 +1138,13 @@ fn contains_any( false } -/// Adds the given `frames` to the given `regions_list` and `frames_list` as a chunk of reserved frames. -/// -/// Returns the range of **new** frames that were added to the lists, +/// Adds the given `frames` to the given `regions_list` and `frames_list` as a chunk of reserved frames. +/// +/// Returns the range of **new** frames that were added to the lists, /// which will be a subset of the given input `frames`. /// /// Currently, this function adds no new frames at all if any frames within the given `frames` list -/// overlap any existing regions at all. +/// overlap any existing regions at all. /// TODO: handle partially-overlapping regions by extending existing regions on either end. fn add_reserved_region_to_lists( regions_list: &mut StaticArrayRBTree, @@ -1129,7 +1162,7 @@ fn add_reserved_region_to_lists( match &mut frames_list.0 { Inner::Array(ref mut arr) => { for chunk in arr.iter().flatten() { - if let Some(_overlap) = chunk.overlap(&frames) { + if let Some(_overlap) = chunk.range().unwrap().overlap(&frames) { // trace!("Failed to add reserved region {:?} due to overlap {:?} with existing chunk {:?}", // frames, _overlap, chunk // ); @@ -1145,7 +1178,7 @@ fn add_reserved_region_to_lists( // so we can stop looking for overlapping regions once we pass the end of the new frames to add. break; } - if let Some(_overlap) = chunk.overlap(&frames) { + if let Some(_overlap) = chunk.range().unwrap().overlap(&frames) { // trace!("Failed to add reserved region {:?} due to overlap {:?} with existing chunk {:?}", // frames, _overlap, chunk // ); @@ -1172,25 +1205,25 @@ fn add_reserved_region_to_lists( /// The core frame allocation routine that allocates the given number of physical frames, /// optionally at the requested starting `PhysicalAddress`. -/// -/// This simply reserves a range of frames; it does not perform any memory mapping. +/// +/// This simply reserves a range of frames; it does not perform any memory mapping. /// Thus, the memory represented by the returned `AllocatedFrames` isn't directly accessible /// until you map virtual pages to them. -/// +/// /// Allocation is based on a red-black tree and is thus `O(log(n))`. /// Fragmentation isn't cleaned up until we're out of address space, but that's not really a big deal. -/// +/// /// # Arguments /// * `requested_paddr`: if `Some`, the returned `AllocatedFrames` will start at the `Frame` -/// containing this `PhysicalAddress`. +/// containing this `PhysicalAddress`. /// If `None`, the first available `Frame` range will be used, starting at any random physical address. -/// * `num_frames`: the number of `Frame`s to be allocated. -/// +/// * `num_frames`: the number of `Frame`s to be allocated. +/// /// # Return /// If successful, returns a tuple of two items: /// * the frames that were allocated, and -/// * an opaque struct representing details of bookkeeping-related actions that may cause heap allocation. -/// Those actions are deferred until this returned `DeferredAllocAction` struct object is dropped, +/// * an opaque struct representing details of bookkeeping-related actions that may cause heap allocation. +/// Those actions are deferred until this returned `DeferredAllocAction` struct object is dropped, /// allowing the caller (such as the heap implementation itself) to control when heap allocation may occur. pub fn allocate_frames_deferred( requested_paddr: Option, @@ -1200,7 +1233,7 @@ pub fn allocate_frames_deferred( warn!("frame_allocator: requested an allocation of 0 frames... stupid!"); return Err("cannot allocate zero frames"); } - + if let Some(paddr) = requested_paddr { let start_frame = Frame::containing_address(paddr); let mut free_reserved_frames_list = FREE_RESERVED_FRAMES_LIST.lock(); @@ -1246,9 +1279,9 @@ pub fn allocate_frames_deferred( /// Similar to [`allocated_frames_deferred()`](fn.allocate_frames_deferred.html), -/// but accepts a size value for the allocated frames in number of bytes instead of number of frames. -/// -/// This function still allocates whole frames by rounding up the number of bytes. +/// but accepts a size value for the allocated frames in number of bytes instead of number of frames. +/// +/// This function still allocates whole frames by rounding up the number of bytes. pub fn allocate_frames_by_bytes_deferred( requested_paddr: Option, num_bytes: usize, @@ -1341,7 +1374,7 @@ where FramesIteratorRequest::AllocateAt { requested_frame, num_frames } => { frame_alloc_request = Some((requested_frame, num_frames)); break; - } + } } } @@ -1359,10 +1392,10 @@ where /// Converts the frame allocator from using static memory (a primitive array) to dynamically-allocated memory. -/// -/// Call this function once heap allocation is available. +/// +/// Call this function once heap allocation is available. /// Calling this multiple times is unnecessary but harmless, as it will do nothing after the first invocation. -#[doc(hidden)] +#[doc(hidden)] pub fn convert_frame_allocator_to_heap_based() { FREE_GENERAL_FRAMES_LIST.lock().convert_to_heap_allocated(); FREE_RESERVED_FRAMES_LIST.lock().convert_to_heap_allocated(); @@ -1370,8 +1403,8 @@ pub fn convert_frame_allocator_to_heap_based() { RESERVED_REGIONS.lock().convert_to_heap_allocated(); } -/// A debugging function used to dump the full internal state of the frame allocator. -#[doc(hidden)] +/// A debugging function used to dump the full internal state of the frame allocator. +#[doc(hidden)] pub fn dump_frame_allocator_state() { debug!("----------------- FREE GENERAL FRAMES ---------------"); FREE_GENERAL_FRAMES_LIST.lock().iter().for_each(|e| debug!("\t {:?}", e) ); diff --git a/kernel/memory/src/lib.rs b/kernel/memory/src/lib.rs index cbd2e7d6b0..82a7732c8d 100644 --- a/kernel/memory/src/lib.rs +++ b/kernel/memory/src/lib.rs @@ -40,6 +40,7 @@ pub use page_allocator::{ pub use frame_allocator::{ AllocatedFrames, UnmappedFrames, + AllocatedFrame, allocate_frames_deferred, allocate_frames_by_bytes_deferred, allocate_frames, diff --git a/kernel/memory/src/paging/mapper.rs b/kernel/memory/src/paging/mapper.rs index 565407321f..ef0ed7641b 100644 --- a/kernel/memory/src/paging/mapper.rs +++ b/kernel/memory/src/paging/mapper.rs @@ -18,8 +18,8 @@ use core::{ slice, }; use log::{error, warn, debug, trace}; -use memory_structs::{PageSize, Page4K}; -use crate::{BROADCAST_TLB_SHOOTDOWN_FUNC, VirtualAddress, PhysicalAddress, Page, Frame, FrameRange, AllocatedPages, AllocatedFrames, UnmappedFrames}; +use memory_structs::{PageSize, Page4K, Page2M, Page1G, MemChunkSize}; +use crate::{BROADCAST_TLB_SHOOTDOWN_FUNC, VirtualAddress, PhysicalAddress, Page, Frame, PageRange, FrameRange, AllocatedPages, AllocatedFrames, UnmappedFrames, AllocatedFrame}; use crate::paging::{ get_current_p4, table::{P4, UPCOMING_P4, Table, Level4}, @@ -265,33 +265,81 @@ impl Mapper { // Only the lowest-level P1 entry can be considered exclusive, and only when // we are mapping it exclusively (i.e., owned `AllocatedFrames` are passed in). - let actual_flags = flags + let mut actual_flags = flags .valid(true) .exclusive(BF::OWNED); let pages_count = pages.size_in_pages(); let frames_count = frames.borrow().size_in_frames(); - if pages_count != frames_count { - error!("map_allocated_pages_to(): pages {:?} count {} must equal frames {:?} count {}!", - pages, pages_count, frames.borrow(), frames_count - ); - return Err("map_allocated_pages_to(): page count must equal frame count"); - } - // TODO FIXME: implement huge pages here. + // Select correct mapping method. + // The different branches are mostly the same. For huge pages an additional flag is set, and + // the frame is mapped to the page table level corresponding page size. + match pages.page_size() { + MemChunkSize::Normal4K => { + // This check is dependent on the page size until size-awareness is added to Frames + if pages_count != frames_count { + error!("map_allocated_pages_to(): pages {:?} count {} must equal frames {:?} count {}!", + pages, pages_count, frames.borrow(), frames_count + ); + return Err("map_allocated_pages_to(): page count must equal frame count"); + } + + // iterate over pages and frames in lockstep + for (page, frame) in pages.range().clone().into_iter().zip(frames.borrow().into_iter()) { + let p3 = self.p4_mut().next_table_create(page.p4_index(), higher_level_flags); + let p2 = p3.next_table_create(page.p3_index(), higher_level_flags); + let p1 = p2.next_table_create(page.p2_index(), higher_level_flags); + if !p1[page.p1_index()].is_unused() { + error!("map_allocated_pages_to(): page {:#X} -> frame {:#X}, page was already in use!", page.start_address(), frame.start_address()); + return Err("map_allocated_pages_to(): page was already in use"); + } + + p1[page.p1_index()].set_entry(frame, actual_flags); + } + } + MemChunkSize::Huge2M => { + if pages_count != frames_count { + error!("map_allocated_pages_to(): pages {:?} count {} must equal frames {:?} count {}!", + pages, pages_count, frames.borrow(), frames_count + ); + return Err("map_allocated_pages_to(): page count must equal frame count"); + } + // Temporarily define a custom step over the page range until correct behaviour is implemented for huge pages + for (page, frame) in pages.range_2mb().clone().into_iter().zip(frames.borrow().into_iter() /*into_iter().step_by(512)*/) { + actual_flags = actual_flags.huge(true); + let p3 = self.p4_mut().next_table_create(page.p4_index(), higher_level_flags); + let p2 = p3.next_table_create(page.p3_index(), higher_level_flags); + + if !p2[page.p2_index()].is_unused() { + error!("map_allocated_pages_to(): page {:#X} -> frame {:#X}, page was already in use!", page.start_address(), frame.start_address()); + return Err("map_allocated_pages_to(): page was already in use"); + } - // iterate over pages and frames in lockstep - for (page, frame) in pages.range().clone().into_iter().zip(frames.borrow().into_iter()) { - let p3 = self.p4_mut().next_table_create(page.p4_index(), higher_level_flags); - let p2 = p3.next_table_create(page.p3_index(), higher_level_flags); - let p1 = p2.next_table_create(page.p2_index(), higher_level_flags); + // let af = Frame::::from(*frame.start()); + p2[page.p2_index()].set_entry(frame ,actual_flags); + } + } + MemChunkSize::Huge1G => { + if pages_count * (512 * 512) != frames_count { + error!("map_allocated_pages_to(): pages {:?} count {} must equal frames {:?} count {}!", + pages, pages_count, frames.borrow(), frames_count + ); + return Err("map_allocated_pages_to(): page count must equal frame count"); + } + // Temporarily define a custom step over the page range until correct behaviour is implemented for huge pages + for (page, frame) in pages.range_1gb().clone().into_iter().zip(frames.borrow().into_iter().step_by(512 * 512)) { + actual_flags = actual_flags.huge(true); + let p3 = self.p4_mut().next_table_create(page.p4_index(), higher_level_flags); - if !p1[page.p1_index()].is_unused() { - error!("map_allocated_pages_to(): page {:#X} -> frame {:#X}, page was already in use!", page.start_address(), frame.start_address()); - return Err("map_allocated_pages_to(): page was already in use"); - } + if !p3[page.p3_index()].is_unused() { + error!("map_allocated_pages_to(): page {:#X} -> frame {:#X}, page was already in use!", page.start_address(), frame.start_address()); + return Err("map_allocated_pages_to(): page was already in use"); + } - p1[page.p1_index()].set_entry(frame, actual_flags); + p3[page.p3_index()].set_entry(frame, actual_flags); + } + } } Ok(( @@ -351,22 +399,31 @@ impl Mapper { .valid(true) .exclusive(true); - for page in pages.range().clone() { - let af = frame_allocator::allocate_frames(1).ok_or("map_allocated_pages(): couldn't allocate new frame, out of memory")?; - - let p3 = self.p4_mut().next_table_create(page.p4_index(), higher_level_flags); - let p2 = p3.next_table_create(page.p3_index(), higher_level_flags); - let p1 = p2.next_table_create(page.p2_index(), higher_level_flags); - - if !p1[page.p1_index()].is_unused() { - error!("map_allocated_pages(): page {:#X} -> frame {:#X}, page was already in use!", - page.start_address(), af.start_address() - ); - return Err("map_allocated_pages(): page was already in use"); - } - - p1[page.p1_index()].set_entry(af.as_allocated_frame(), actual_flags); - core::mem::forget(af); // we currently forget frames allocated here since we don't yet have a way to track them. + match pages.page_size() { + MemChunkSize::Normal4K => { + for page in pages.range().clone() { + let af = frame_allocator::allocate_frames(1).ok_or("map_allocated_pages(): couldn't allocate new frame, out of memory")?; + let p3 = self.p4_mut().next_table_create(page.p4_index(), higher_level_flags); + let p2 = p3.next_table_create(page.p3_index(), higher_level_flags); + let p1 = p2.next_table_create(page.p2_index(), higher_level_flags); + + if !p1[page.p1_index()].is_unused() { + error!("map_allocated_pages(): page {:#X} -> frame {:#X}, page was already in use!", + page.start_address(), af.start_address() + ); + return Err("map_allocated_pages(): page was already in use"); + } + + p1[page.p1_index()].set_entry(af.as_allocated_frame(), actual_flags); + core::mem::forget(af); // we currently forget frames allocated here since we don't yet have a way to track them. + } + } + MemChunkSize::Huge2M => { + todo!("Mapping 2MiB huge pages to randomly-allocated huge frames is not yet supported") + } + MemChunkSize::Huge1G => { + todo!("Mapping 1GiB huge pages to randomly-allocated huge frames is not yet supported") + } } Ok(MappedPages { @@ -685,18 +742,44 @@ impl MappedPages { return Ok(()); } - for page in self.pages.range().clone() { - let p1 = active_table_mapper.p4_mut() - .next_table_mut(page.p4_index()) - .and_then(|p3| p3.next_table_mut(page.p3_index())) - .and_then(|p2| p2.next_table_mut(page.p2_index())) - .ok_or("mapping code does not support huge pages")?; - - p1[page.p1_index()].set_flags(new_flags); + match self.pages.page_size() { + MemChunkSize::Normal4K => { + for page in self.pages.range().clone() { + let p1 = active_table_mapper.p4_mut() + .next_table_mut(page.p4_index()) + .and_then(|p3| p3.next_table_mut(page.p3_index())) + .and_then(|p2| p2.next_table_mut(page.p2_index())) + .ok_or("BUG: remap() - could not get p1 entry for 4kb page")?; + + p1[page.p1_index()].set_flags(new_flags); + + tlb_flush_virt_addr(page.start_address()); + } + } + MemChunkSize::Huge2M => { + for page in self.pages.range_2mb().clone().into_iter() { + let p2 = active_table_mapper.p4_mut() + .next_table_mut(page.p4_index()) + .and_then(|p3| p3.next_table_mut(page.p3_index())) + .ok_or("BUG: remap() - could not get p1 entry for 2mb page")?; + + p2[page.p2_index()].set_flags(new_flags); - tlb_flush_virt_addr(page.start_address()); + tlb_flush_virt_addr(page.start_address()); + } + } + MemChunkSize::Huge1G => { + for page in self.pages.range_1gb().clone().into_iter() { + let p3 = active_table_mapper.p4_mut() + .next_table_mut(page.p4_index()) + .ok_or("BUG: remap() - could not get p1 entry for 1gb page")?; + + p3[page.p3_index()].set_flags(new_flags); + + tlb_flush_virt_addr(page.start_address()); + } + } } - if let Some(func) = BROADCAST_TLB_SHOOTDOWN_FUNC.get() { func(self.pages.range().clone()); } @@ -756,75 +839,191 @@ impl MappedPages { "BUG: MappedPages::unmap(): current P4 must equal original P4, \ cannot unmap MappedPages from a different page table than they were originally mapped to!" ); - } + } let mut first_frame_range: Option = None; // this is what we'll return let mut current_frame_range: Option = None; - for page in self.pages.range().clone() { - let p1 = active_table_mapper.p4_mut() - .next_table_mut(page.p4_index()) - .and_then(|p3| p3.next_table_mut(page.p3_index())) - .and_then(|p2| p2.next_table_mut(page.p2_index())) - .ok_or("mapping code does not support huge pages")?; - let pte = &mut p1[page.p1_index()]; - if pte.is_unused() { - return Err("unmap(): page not mapped"); - } - - let unmapped_frames = pte.set_unmapped(); - tlb_flush_virt_addr(page.start_address()); - - // Here, create (or extend) a contiguous ranges of frames here based on the `unmapped_frames` - // freed from the newly-unmapped P1 PTE entry above. - match unmapped_frames { - UnmapResult::Exclusive(newly_unmapped_frames) => { - let newly_unmapped_frames = INTO_UNMAPPED_FRAMES_FUNC.get() - .ok_or("BUG: Mapper::unmap(): the `INTO_UNMAPPED_FRAMES_FUNC` callback was not initialized") - .map(|into_func| into_func(newly_unmapped_frames.deref().clone()))?; - - if let Some(mut curr_frames) = current_frame_range.take() { - match curr_frames.merge(newly_unmapped_frames) { - Ok(()) => { - // Here, the newly unmapped frames were contiguous with the current frame_range, - // and we successfully merged them into a single range of AllocatedFrames. - current_frame_range = Some(curr_frames); - } - Err(newly_unmapped_frames) => { - // Here, the newly unmapped frames were **NOT** contiguous with the current_frame_range, - // so we "finish" the current_frame_range (it's already been "taken") and start a new one - // based on the newly unmapped frames. + // Select the correct unmapping behaviour based on page size. + // The different branches mostly have the same logic, differing + // only in what level is unmapped and what unmapping function is used. + match self.pages.page_size() { + MemChunkSize::Normal4K => { + for page in self.pages.range().clone() { + let p1 = active_table_mapper.p4_mut() + .next_table_mut(page.p4_index()) + .and_then(|p3| p3.next_table_mut(page.p3_index())) + .and_then(|p2| p2.next_table_mut(page.p2_index())) + .ok_or("BUG: could not get p1 entry in unmap()")?; + let pte = &mut p1[page.p1_index()]; + if pte.is_unused() { + return Err("unmap(): page not mapped"); + } + + let unmapped_frames = pte.set_unmapped(); + tlb_flush_virt_addr(page.start_address()); + + // Here, create (or extend) a contiguous ranges of frames here based on the `unmapped_frames` + // freed from the newly-unmapped P1 PTE entry above. + match unmapped_frames { + UnmapResult::Exclusive(newly_unmapped_frames) => { + let newly_unmapped_frames = INTO_UNMAPPED_FRAMES_FUNC.get() + .ok_or("BUG: Mapper::unmap(): the `INTO_UNMAPPED_FRAMES_FUNC` callback was not initialized") + .map(|into_func| into_func(newly_unmapped_frames.deref().clone()))?; + + if let Some(mut curr_frames) = current_frame_range.take() { + match curr_frames.merge(newly_unmapped_frames) { + Ok(()) => { + // Here, the newly unmapped frames were contiguous with the current frame_range, + // and we successfully merged them into a single range of AllocatedFrames. + current_frame_range = Some(curr_frames); + } + Err(newly_unmapped_frames) => { + // Here, the newly unmapped frames were **NOT** contiguous with the current_frame_range, + // so we "finish" the current_frame_range (it's already been "taken") and start a new one + // based on the newly unmapped frames. + current_frame_range = Some(newly_unmapped_frames); + + // If this is the first frame range we've unmapped, don't drop it -- save it as the return value. + if first_frame_range.is_none() { + first_frame_range = Some(curr_frames); + } else { + // If this is NOT the first frame range we've unmapped, then go ahead and drop it now, + // otherwise there will not be any other opportunity for it to be dropped. + // + // TODO: here in the future, we could add it to the optional input list (see this function's doc comments) + // of AllocatedFrames to return, i.e., `Option<&mut Vec>`. + trace!("MappedPages::unmap(): dropping additional non-contiguous frames {:?}", curr_frames); + // curr_frames is dropped here + } + } + } + } else { + // This was the first frames we unmapped, so start a new current_frame_range. current_frame_range = Some(newly_unmapped_frames); - - // If this is the first frame range we've unmapped, don't drop it -- save it as the return value. - if first_frame_range.is_none() { - first_frame_range = Some(curr_frames); - } else { - // If this is NOT the first frame range we've unmapped, then go ahead and drop it now, - // otherwise there will not be any other opportunity for it to be dropped. - // - // TODO: here in the future, we could add it to the optional input list (see this function's doc comments) - // of AllocatedFrames to return, i.e., `Option<&mut Vec>`. - trace!("MappedPages::unmap(): dropping additional non-contiguous frames {:?}", curr_frames); - // curr_frames is dropped here + } + } + UnmapResult::NonExclusive(_frames) => { + // trace!("Note: FYI: page {:X?} -> frames {:X?} was just unmapped but not mapped as EXCLUSIVE.", page, _frames); + } + } + } + + #[cfg(not(bm_map))] + { + if let Some(func) = BROADCAST_TLB_SHOOTDOWN_FUNC.get() { + func(self.pages.range().clone()); + } + } + } + MemChunkSize::Huge2M => { + // Temporarily define a custom step over huge page ranges until correct behaiour is implemented + for page in self.pages.range_2mb().clone().into_iter() { + let p2 = active_table_mapper.p4_mut() + .next_table_mut(page.p4_index()) + .and_then(|p3| p3.next_table_mut(page.p3_index())) + .ok_or("BUG: could not get p2 entry for 2mb page in unmap()")?; + let pte = &mut p2[page.p2_index()]; + if pte.is_unused() { + return Err("unmap(): page not mapped"); + } + let unmapped_frames = pte.set_unmapped_2mb(); + tlb_flush_virt_addr(page.start_address()); + + match unmapped_frames { + UnmapResult::Exclusive(newly_unmapped_frames) => { + let newly_unmapped_frames = INTO_UNMAPPED_FRAMES_FUNC.get() + .ok_or("BUG: Mapper::unmap(): the `INTO_UNMAPPED_FRAMES_FUNC` callback was not initialized") + .map(|into_func| into_func(newly_unmapped_frames.deref().clone()))?; + if let Some(mut curr_frames) = current_frame_range.take() { + match curr_frames.merge(newly_unmapped_frames) { + Ok(()) => { + current_frame_range = Some(curr_frames); + } + Err(newly_unmapped_frames) => { + current_frame_range = Some(newly_unmapped_frames); + + if first_frame_range.is_none() { + first_frame_range = Some(curr_frames); + } else { + // TODO: here in the future, we could add it to the optional input list (see this function's doc comments) + // of AllocatedFrames to return, i.e., `Option<&mut Vec>`. + trace!("MappedPages::unmap(): dropping additional non-contiguous frames {:?}", curr_frames); + // curr_frames is dropped here + } + } } + } else { + current_frame_range = Some(newly_unmapped_frames); } } - } else { - // This was the first frames we unmapped, so start a new current_frame_range. - current_frame_range = Some(newly_unmapped_frames); + UnmapResult::NonExclusive(_frames) => { + //trace!("Note: FYI: page {:X?} -> frames {:X?} was just unmapped but not mapped as EXCLUSIVE.", page, _frames); + } } } - UnmapResult::NonExclusive(_frames) => { - // trace!("Note: FYI: page {:X?} -> frames {:X?} was just unmapped but not mapped as EXCLUSIVE.", page, _frames); + + #[cfg(not(bm_map))] + { + if let Some(func) = BROADCAST_TLB_SHOOTDOWN_FUNC.get() { + func(PageRange::::from(self.pages.range_2mb())); // convert to 4kb range for the TLB shootdown + } } } - } - - #[cfg(not(bm_map))] - { - if let Some(func) = BROADCAST_TLB_SHOOTDOWN_FUNC.get() { - func(self.pages.range().clone()); + MemChunkSize::Huge1G => { + // Temporarily define a custom step over huge page ranges until correct behaiour is implemented + for page in self.pages.range_1gb().clone().into_iter() { + let p3 = active_table_mapper.p4_mut() + .next_table_mut(page.p4_index()) + .ok_or("BUG: could not get p2 entry for 2gb page in unmap()")?; + let pte = &mut p3[page.p3_index()]; + if pte.is_unused() { + return Err("unmap(): page not mapped"); + } + + let unmapped_frames = pte.set_unmapped_1gb(); + tlb_flush_virt_addr(page.start_address()); + + match unmapped_frames { + UnmapResult::Exclusive(newly_unmapped_frames) => { + let newly_unmapped_frames = INTO_UNMAPPED_FRAMES_FUNC.get() + .ok_or("BUG: Mapper::unmap(): the `INTO_UNMAPPED_FRAMES_FUNC` callback was not initialized") + .map(|into_func| into_func(newly_unmapped_frames.deref().clone()))?; + + if let Some(mut curr_frames) = current_frame_range.take() { + match curr_frames.merge(newly_unmapped_frames) { + Ok(()) => { + current_frame_range = Some(curr_frames); + } + Err(newly_unmapped_frames) => { + current_frame_range = Some(newly_unmapped_frames); + + if first_frame_range.is_none() { + first_frame_range = Some(curr_frames); + } else { + // TODO: here in the future, we could add it to the optional input list (see this function's doc comments) + // of AllocatedFrames to return, i.e., `Option<&mut Vec>`. + trace!("MappedPages::unmap(): dropping additional non-contiguous frames {:?}", curr_frames); + // curr_frames is dropped here + } + } + } + } else { + current_frame_range = Some(newly_unmapped_frames); + } + } + UnmapResult::NonExclusive(_frames) => { + // trace!("Note: FYI: page {:X?} -> frames {:X?} was just unmapped but not mapped as EXCLUSIVE.", page, _frames); + } + } + } + + #[cfg(not(bm_map))] + { + if let Some(func) = BROADCAST_TLB_SHOOTDOWN_FUNC.get() { + func(PageRange::::from(self.pages.range_1gb())); // convert to 4kb range for the TLB shootdown + } + } } } @@ -833,7 +1032,6 @@ impl MappedPages { .or(current_frame_range.map(|f| f.into_allocated_frames()))) } - /// Reinterprets this `MappedPages`'s underlying memory region as a struct of the given type `T`, /// i.e., overlays a struct on top of this mapped memory region. /// diff --git a/kernel/memory_structs/src/lib.rs b/kernel/memory_structs/src/lib.rs index 6d0bef8625..82dd24ba21 100644 --- a/kernel/memory_structs/src/lib.rs +++ b/kernel/memory_structs/src/lib.rs @@ -3,7 +3,7 @@ //! The types of interest are divided into three categories: //! 1. addresses: `VirtualAddress` and `PhysicalAddress`. //! 2. "chunk" types: `Page` and `Frame`. -//! 3. ranges of chunks: `PageRange` and `FrameRange`. +//! 3. ranges of chunks: `PageRange` and `FrameRange`. #![no_std] #![feature(step_trait)] @@ -100,9 +100,9 @@ macro_rules! implement_address { #[doc = "A " $desc " memory address, which is a `usize` under the hood."] #[derive( - Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, - Binary, Octal, LowerHex, UpperHex, - BitAnd, BitOr, BitXor, BitAndAssign, BitOrAssign, BitXorAssign, + Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, + Binary, Octal, LowerHex, UpperHex, + BitAnd, BitOr, BitXor, BitAndAssign, BitOrAssign, BitXorAssign, Add, Sub, AddAssign, SubAssign, FromBytes, )] @@ -111,7 +111,7 @@ macro_rules! implement_address { impl $TypeName { #[doc = "Creates a new `" $TypeName "`, returning an error if the address is not canonical.\n\n \ - This is useful for checking whether an address is valid before using it. + This is useful for checking whether an address is valid before using it. For example, on x86_64, virtual addresses are canonical if their upper bits `(64:48]` are sign-extended from bit 47, and physical addresses are canonical if their upper bits `(64:52]` are 0."] @@ -509,21 +509,30 @@ macro_rules! implement_page_frame { } } impl From<$TypeName> for $TypeName { - fn from(p: $TypeName) -> Self { - Self { + fn from(p: $TypeName) -> Self { + Self { number: p.number, size: PhantomData } } } impl From<$TypeName> for $TypeName { - fn from(p: $TypeName) -> Self { - Self { + fn from(p: $TypeName) -> Self { + Self { number: p.number, size: PhantomData } } } + + impl $TypeName

{ + pub fn convert_to_4k(&self) -> $TypeName { + $TypeName { + number: self.number, + size: PhantomData + } + } + } } }; } @@ -564,7 +573,6 @@ impl Page

{ macro_rules! implement_page_frame_range { ($TypeName:ident, $desc:literal, $short:ident, $chunk:ident, $address:ident) => { paste! { // using the paste crate's macro for easy concatenation - #[doc = "A range of [`" $chunk "`]s that are contiguous in " $desc " memory."] #[derive(Clone, PartialEq, Eq)] pub struct $TypeName(RangeInclusive<$chunk::

>); @@ -737,6 +745,24 @@ macro_rules! implement_page_frame_range { self.0.iter() } } + + + #[doc = "A `" $TypeName "` that implements `Copy`."] + #[derive(Clone, Copy)] + pub struct [] { + start: $chunk

, + end: $chunk

, + } + impl From<$TypeName

> for []

{ + fn from(r: $TypeName

) -> Self { + Self { start: *r.start(), end: *r.end() } + } + } + impl From<[]

> for $TypeName

{ + fn from(cr: []

) -> Self { + Self::new(cr.start, cr.end) + } + } impl IntoIterator for $TypeName

{ type Item = $chunk

; type IntoIter = RangeInclusiveIterator<$chunk

>; @@ -781,6 +807,230 @@ macro_rules! implement_page_frame_range { } } } + + #[doc = "An enum used to wrap the generic `" $TypeName "` variants corresponding to different `" $chunk "` sizes. \ + Additional methods are provided in order to destructure the enum variants."] + #[derive(Debug, Clone, PartialEq, Eq)] + pub enum [<$TypeName Sized>] { + Normal4KiB($TypeName), + Huge2MiB($TypeName), + Huge1GiB($TypeName), + } + + impl [<$TypeName Sized>] { + #[doc = "Get the size of the pages for the contained `" $TypeName"` ." + pub fn page_size(&self) -> MemChunkSize { + match self { + Self::Normal4KiB(pr) => { + pr.start().page_size() + } + Self::Huge2MiB(pr) => { + pr.start().page_size() + } + Self::Huge1GiB(pr) => { + pr.start().page_size() + } + } + } + + #[doc = "Returns a reference to the contained `" $TypeName "` holding 4kb `" $chunk "`s. Returns None if called on a `" $TypeName "` holding huge pages."] + pub fn range(&self) -> Option<&$TypeName> { + match self { + Self::Normal4KiB(pr) => { + Some(pr) + } + _ => { + None + } + } + } + + #[doc = "range() equivalent for 2MiB memory ranges"] + pub fn range_2mb(&self) -> Result<$TypeName, &'static str> { + match self { + Self::Huge2MiB(pr) => { + Ok(pr.clone()) + } + _ => { + Err("Called range_2mb on a $TypeName with a size other than 2mb") + } + } + } + + #[doc = "range() equivalent for 1GiB memory ranges"] + pub fn range_1gb(&self) -> Result<$TypeName, &'static str> { + match self { + Self::Huge1GiB(pr) => { + Ok(pr.clone()) + } + _ => { + Err("Called range_1gb on a $TypeName with a size other than 1gb") + } + } + } + + pub fn contains(&self, page: &$chunk) -> bool { + match self { + Self::Normal4KiB(pr) => { + pr.contains(page) + } + // 'page' is a 4kb chunk, so we need to perform a temporary conversion for other sizes + Self::Huge2MiB(pr) => { + let pr_4k = $TypeName::::from(pr.clone()); + pr_4k.contains(page) + } + Self::Huge1GiB(pr) => { + let pr_4k = $TypeName::::from(pr.clone()); + pr_4k.contains(page) + } + } + } + + pub const fn offset_of_address(&self, addr: $address) -> Option { + match self { + Self::Normal4KiB(pr) => { + pr.offset_of_address(addr) + } + Self::Huge2MiB(pr) => { + pr.offset_of_address(addr) + } + Self::Huge1GiB(pr) => { + pr.offset_of_address(addr) + } + } + } + + pub const fn address_at_offset(&self, offset: usize) -> Option<$address> { + match self { + Self::Normal4KiB(pr) => { + pr.address_at_offset(offset) + } + Self::Huge2MiB(pr) => { + pr.address_at_offset(offset) + } + Self::Huge1GiB(pr) => { + pr.address_at_offset(offset) + } + } + } + + #[doc = "Returns the starting `" $address "` in this range."] + pub fn start_address(&self) -> $address { + match self { + Self::Normal4KiB(pr) => { + pr.start_address() + } + Self::Huge2MiB(pr) => { + pr.start_address() + } + Self::Huge1GiB(pr) => { + pr.start_address() + } + } + } + + #[doc = "Returns the size in bytes of this range."] + pub fn size_in_bytes(&self) -> usize { + match self { + Self::Normal4KiB(pr) => { + pr.size_in_bytes() + } + Self::Huge2MiB(pr) => { + pr.size_in_bytes() + } + Self::Huge1GiB(pr) => { + pr.size_in_bytes() + } + } + } + + #[doc = "Returns the size, in number of `" $chunk "`s, of this range."] + pub fn [](&self) -> usize { + match self { + Self::Normal4KiB(pr) => { + pr.[]() + } + Self::Huge2MiB(pr) => { + pr.[]() + } + Self::Huge1GiB(pr) => { + pr.[]() + } + } + } + + #[doc = "Returns the starting `" $chunk" ` in this range. TODO: Find an alternative to panic when called on wrong size." + pub fn start(&self) -> &$chunk { + match self { + Self::Normal4KiB(pr) => { + pr.start() + } + _ => { + panic!("Attempt to get the start of a huge page range as a 4KiB page."); + } + } + } + + #[doc = "Returns the ending `" $chunk "` (inclusive) in this range. TODO: Find an alternative to panic when called on wrong size."] + pub fn end(&self) -> &$chunk { + match self { + Self::Normal4KiB(pr) => { + pr.end() + } + _ => { + panic!("Attempt to get the end of a huge page range as a 4KiB page."); + } + } + } + + #[doc = "start() equivalent for 2mb `" $TypeName "`s. TODO: Find an alternative to panic when called on wrong size."] + pub fn start_2m(&self) -> &$chunk { + match self { + Self::Huge2MiB(pr) => { + pr.start() + } + _ => { + panic!("Attempt to get the start of a huge page range as a 4KiB page."); + } + } + } + + #[doc = "start() equivalent for 2mb `" $TypeName "`s. TODO: Find an alternative to panic when called on wrong size."] + pub fn end_2m(&self) -> &$chunk { + match self { + Self::Huge2MiB(pr) => { + pr.end() + } + _ => { + panic!("Attempt to get the end of a huge page range as a 4KiB page."); + } + } + } + + #[doc = "start() equivalent for 1gb `" $TypeName "`s. TODO: Find an alternative to panic when called on wrong size."] + pub fn start_1g(&self) -> &$chunk { + match self { + Self::Huge1GiB(pr) => { + pr.start() + } + _ => { + panic!("Attempt to get the start of a huge page range as a 4KiB page."); + } + } + } + + #[doc = "start() equivalent for 1gb `" $TypeName "`s. TODO: Find an alternative to panic when called on wrong size."] + pub fn end_1g(&self) -> &$chunk { + match self { + Self::Huge1GiB(pr) => { + pr.end() + } + _ => { + panic!("Attempt to get the end of a huge page range as a 4KiB page."); + } + } + } + } } }; } diff --git a/kernel/page_allocator/src/lib.rs b/kernel/page_allocator/src/lib.rs index 558c8d7d28..9a69de0964 100644 --- a/kernel/page_allocator/src/lib.rs +++ b/kernel/page_allocator/src/lib.rs @@ -30,7 +30,7 @@ use intrusive_collections::Bound; mod static_array_rb_tree; // mod static_array_linked_list; -use core::{borrow::Borrow, cmp::{Ordering, max, min}, fmt, ops::{Deref, DerefMut}}; +use core::{borrow::Borrow, cmp::{Ordering, max, min}, fmt, ops::{Deref, DerefMut}, convert::TryFrom}; use kernel_config::memory::*; use memory_structs::{VirtualAddress, Page, PageRange, PageSize, Page4K, Page2M, Page1G}; use spin::{Mutex, Once}; @@ -351,6 +351,31 @@ impl AllocatedPages

{ AllocatedPages::

{ pages: second }, )) } + + /// Returns the size of the pages in this page range. + pub fn page_size(&self) -> MemChunkSize { + self.pages.page_size() + } + + /// Converts a range of 4kb `AllocatedPages` into a range of 2mb `AllocatedPages`. + pub fn to_2mb_allocated_pages(&mut self) { + self.pages = PageRangeSized::Huge2MiB( + PageRange::::try_from( + self.pages + .range() + .unwrap() + .clone()).unwrap()) + } + + /// Converts a range of 4kb `AllocatedPages` into a range of 1gb `AllocatedPages`. + pub fn to_1gb_allocated_pages(&mut self) { + self.pages = PageRangeSized::Huge1GiB( + PageRange::::try_from( + self.pages + .range() + .unwrap() + .clone()).unwrap()) + } } impl Drop for AllocatedPages

{ @@ -358,6 +383,17 @@ impl Drop for AllocatedPages

{ if self.size_in_pages() == 0 { return; } // trace!("page_allocator: deallocating {:?}", self); + // Convert huge pages back to default size if needed. + let pages = match self.page_size() { + MemChunkSize::Normal4K => self.pages.range().unwrap().clone(), + MemChunkSize::Huge2M => { + PageRange::::from(self.pages.range_2mb().unwrap()) + }, + MemChunkSize::Huge1G => { + PageRange::::from(self.pages.range_1gb().unwrap()) + } + }; + let chunk = Chunk { pages: self.pages.clone().into_4k_pages(), }; @@ -917,6 +953,45 @@ pub fn allocate_pages_by_bytes_in_range( .map(|(ap, _action)| ap) } +/// Allocates the given number of 2MB huge pages with no constraints on the starting virtual address. +/// +/// See [`allocate_pages_deferred()`](fn.allocate_pages_deferred.html) for more details. +pub fn allocate_2mb_pages(num_pages: usize) -> Option { + let huge_num_pages = num_pages * 512; + let ap = allocate_pages_deferred(AllocationRequest::AlignedTo { alignment_4k_pages: 512 }, huge_num_pages) + .map(|(ap, _action)| ap) + .ok(); + match ap { + None => { + None + } + Some(mut p) => { // Since this function converts *this* AllocatedPages, it needs to be + // mutable + p.to_2mb_allocated_pages(); + Some(p) + } + } +} + +/// Allocates the given number of 1GB huge pages with no constraints on the starting virtual address. +/// +/// See [`allocate_pages_deferred()`](fn.allocate_pages_deferred.html) for more details. +pub fn allocate_1gb_pages(num_pages: usize) -> Option { + let huge_num_pages = num_pages * 512 * 512; + let ap = allocate_pages_deferred(AllocationRequest::AlignedTo { alignment_4k_pages: 512 * 512 }, huge_num_pages) + .map(|(ap, _action)| ap) + .ok(); + match ap { + None => { + None + } + Some(mut p) => { // Since this function converts *this* AllocatedPages, it needs to be + // mutable + p.to_1gb_allocated_pages(); + Some(p) + } + } +} /// Converts the page allocator from using static memory (a primitive array) to dynamically-allocated memory. /// diff --git a/kernel/page_table_entry/src/lib.rs b/kernel/page_table_entry/src/lib.rs index 07bb703d69..0eb577886a 100644 --- a/kernel/page_table_entry/src/lib.rs +++ b/kernel/page_table_entry/src/lib.rs @@ -63,6 +63,42 @@ impl PageTableEntry { } } + /// Since only 4kb frames are used right now, we can't use the type parameter to get the correct size. + /// This separate function may not be necessary in the future. + pub fn set_unmapped_2mb(&mut self) -> UnmapResult { + let frame = self.frame_value(); + let flags = self.flags(); + self.zero(); + + let frame_range = FrameRange::new( + frame, + Frame::containing_address(frame.start_address() + (4096 * 512))); + + if flags.is_exclusive() { + UnmapResult::Exclusive(UnmappedFrameRange(frame_range)) + } else { + UnmapResult::NonExclusive(frame_range) + } + } + /// Since only 4kb frames are used right now, we can't use the type parameter to get the correct size. + /// This separate function may not be necessary in the future. + pub fn set_unmapped_1gb(&mut self) -> UnmapResult { + let frame = self.frame_value(); + let flags = self.flags(); + self.zero(); + + let frame_range = FrameRange::new( + frame, + Frame::containing_address(frame.start_address() + (4096 * (512 * 512)))); + + if flags.is_exclusive() { + UnmapResult::Exclusive(UnmappedFrameRange(frame_range)) + } else { + UnmapResult::NonExclusive(frame_range) + } + } + + /// Returns this `PageTableEntry`'s flags. pub fn flags(&self) -> PteFlagsArch { PteFlagsArch::from_bits_truncate(self.0 & !PTE_FRAME_MASK) diff --git a/kernel/pte_flags/src/pte_flags_x86_64.rs b/kernel/pte_flags/src/pte_flags_x86_64.rs index c47a6880d0..43543f834f 100644 --- a/kernel/pte_flags/src/pte_flags_x86_64.rs +++ b/kernel/pte_flags/src/pte_flags_x86_64.rs @@ -225,6 +225,16 @@ impl PteFlagsX86_64 { self } + /// Returns a copy of this `PteFlagsX86_64` with the `HUGE` bit set or cleared. + /// + /// * If `enable` is `true`, this page will be treated as a huge page. + /// * If `enable` is `false`, this page will be treated as a page of standard 4KiB size. + #[must_use] + pub fn huge(mut self, enable: bool) -> Self { + self.set(Self::HUGE_PAGE, enable); + self + } + #[doc(alias("present"))] pub const fn is_valid(&self) -> bool { self.contains(Self::VALID) @@ -272,10 +282,10 @@ impl PteFlagsX86_64 { /// * P4, P3, and P2 entries should never set `NOT_EXECUTABLE`, /// only the lowest-level P1 entry should. /// * Clears the `EXCLUSIVE` bit. - /// * Currently, we do not use the `EXCLUSIVE` bit for P4, P3, or P2 entries, + /// * Currently, we only use the `EXCLUSIVE` bit for leaf nodes that directly map a frame in the page table, /// because another page table frame may re-use it (create another alias to it) /// without our page table implementation knowing about it. - /// * Only P1-level PTEs can map a frame exclusively. + /// * Only P1-level PTEs, and in the case of huge pages, P2 and P3-level PTEs, can map a frame exclusively. /// * Clears the PAT index value, as we only support PAT on P1-level PTEs. /// * Sets the `VALID` bit, as every P4, P3, and P2 entry must be valid. #[must_use]