From fff6a7b436267ecee20bfbe44e4814228eedf3cc Mon Sep 17 00:00:00 2001 From: Klimenty Tsoutsman Date: Wed, 27 Dec 2023 16:38:25 +1100 Subject: [PATCH 1/6] Improve `epoch_scheduler` The new scheduler is based on the `O(1)` Linux scheduling algorithm. Signed-off-by: Klimenty Tsoutsman --- Cargo.lock | 5 + kernel/scheduler_epoch/Cargo.toml | 9 +- kernel/scheduler_epoch/src/lib.rs | 208 ++++++++----------- kernel/scheduler_epoch/src/queue.rs | 178 ++++++++++++++++ kernel/scheduler_epoch/src/queue/a.rs | 0 kernel/scheduler_epoch/src/queue/priority.rs | 71 +++++++ libs/bit_set/Cargo.toml | 6 + libs/bit_set/src/iter.rs | 69 ++++++ libs/bit_set/src/lib.rs | 162 +++++++++++++++ 9 files changed, 575 insertions(+), 133 deletions(-) create mode 100644 kernel/scheduler_epoch/src/queue.rs create mode 100644 kernel/scheduler_epoch/src/queue/a.rs create mode 100644 kernel/scheduler_epoch/src/queue/priority.rs create mode 100644 libs/bit_set/Cargo.toml create mode 100644 libs/bit_set/src/iter.rs create mode 100644 libs/bit_set/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c05769a28f..0e12198ed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,6 +256,10 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" +[[package]] +name = "bit_set" +version = "0.1.0" + [[package]] name = "bitfield" version = "0.13.2" @@ -3130,6 +3134,7 @@ dependencies = [ name = "scheduler_epoch" version = "0.1.0" dependencies = [ + "bit_set", "log", "spin 0.9.4", "task", diff --git a/kernel/scheduler_epoch/Cargo.toml b/kernel/scheduler_epoch/Cargo.toml index e0b65c5a15..3a877a6c23 100644 --- a/kernel/scheduler_epoch/Cargo.toml +++ b/kernel/scheduler_epoch/Cargo.toml @@ -6,13 +6,10 @@ version = "0.1.0" edition = "2021" [dependencies] +bit_set = { path = "../../libs/bit_set" } +log = "0.4.8" spin = "0.9.4" - -[dependencies.log] -version = "0.4.8" - -[dependencies.task] -path = "../task" +task = { path = "../task" } [lib] crate-type = ["rlib"] diff --git a/kernel/scheduler_epoch/src/lib.rs b/kernel/scheduler_epoch/src/lib.rs index 22a6917466..a25ed46003 100644 --- a/kernel/scheduler_epoch/src/lib.rs +++ b/kernel/scheduler_epoch/src/lib.rs @@ -1,34 +1,36 @@ -//! This crate implements a token-based epoch scheduling policy. +//! A token-based epoch scheduler. //! -//! At the begining of each scheduling epoch, a set of tokens is distributed -//! among all runnable tasks, based on their priority relative to all other -//! runnable tasks in the runqueue. The formula for this is: -//! ```ignore -//! tokens_assigned_to_task_i = (priority_task_i / sum_priority_all_tasks) * epoch_length; -//! ``` -//! * Each time a task is picked, its token count is decremented by 1. -//! * A task can only be selected for next execution if it has tokens remaining. -//! * When all tokens of all runnable task are exhausted, a new scheduling epoch begins. +//! The implementation is based on the [`O(1)` Linux +//! scheduler][linux-scheduler]. //! -//! This epoch scheduler is also a priority-based scheduler, so it allows -//! getting and setting the priorities of each task. +//! [linux-scheduler]: https://litux.nl/mirror/kerneldevelopment/0672327201/ch04lev1sec2.html #![no_std] +#![feature(core_intrinsics)] + +mod queue; extern crate alloc; -use alloc::{boxed::Box, collections::VecDeque, vec::Vec}; -use core::ops::{Deref, DerefMut}; +use alloc::{boxed::Box, vec::Vec}; +use core::{ + mem, + ops::{Deref, DerefMut}, +}; + use task::TaskRef; -const MAX_PRIORITY: u8 = 40; +use crate::queue::RunQueue; + +const MAX_PRIORITY: u8 = 63; const DEFAULT_PRIORITY: u8 = 20; -const INITIAL_TOKENS: usize = 10; +const INITIAL_TOKENS: usize = 0; /// An instance of an epoch scheduler, typically one per CPU. pub struct Scheduler { idle_task: TaskRef, - queue: VecDeque, + active: RunQueue, + expired: RunQueue, } impl Scheduler { @@ -36,120 +38,73 @@ impl Scheduler { pub const fn new(idle_task: TaskRef) -> Self { Self { idle_task, - queue: VecDeque::new(), + active: RunQueue::new(), + expired: RunQueue::new(), } } - /// Moves the `TaskRef` at the given `index` in this scheduler's runqueue - /// to the end (back) of the runqueue. - /// - /// Sets the number of tokens for that task to the given `tokens` - /// and increments that task's number of context switches. - /// - /// Returns a cloned reference to the `TaskRef` at the given `index`. - fn update_and_move_to_end(&mut self, index: usize, tokens: usize) -> Option { - if let Some(mut priority_task_ref) = self.queue.remove(index) { - priority_task_ref.tokens_remaining = tokens; - let task_ref = priority_task_ref.task.clone(); - self.queue.push_back(priority_task_ref); - Some(task_ref) + fn apply(&mut self, f: F) -> R + where + F: Fn(&mut RunQueue) -> R, + R: Returnable, + { + let (first, second) = if self.active.len() >= self.expired.len() { + (&mut self.active, &mut self.expired) } else { - None - } - } + (&mut self.expired, &mut self.active) + }; - fn try_next(&mut self) -> Option { - if let Some((task_index, _)) = self - .queue - .iter() - .enumerate() - .find(|(_, task)| task.is_runnable() && task.tokens_remaining > 0) - { - let chosen_task = self.queue.get(task_index).unwrap(); - let modified_tokens = chosen_task.tokens_remaining.saturating_sub(1); - self.update_and_move_to_end(task_index, modified_tokens) + let first_result = f(first); + + if first_result.should_return() { + first_result } else { - None + f(second) } } +} - fn assign_tokens(&mut self) { - // We begin with total priorities = 1 to avoid division by zero - let mut total_priorities: usize = 1; - - // This loop calculates the total priorities of the runqueue - for (_i, t) in self.queue.iter().enumerate() { - // we assign tokens only to runnable tasks - if !t.is_runnable() { - continue; - } - - total_priorities = total_priorities - .saturating_add(1) - .saturating_add(t.priority as usize); - } - - // Each epoch lasts for a total of 100 tokens by default. - // However, as this granularity could skip over low priority tasks - // when many concurrent tasks are running, we increase the epoch in such cases. - let epoch: usize = core::cmp::max(total_priorities, 100); - - for (_i, t) in self.queue.iter_mut().enumerate() { - // we give zero tokens to the idle tasks - if t.is_an_idle_task { - continue; - } - - // we give zero tokens to non-runnable tasks - if !t.is_runnable() { - continue; - } +trait Returnable { + fn should_return(&self) -> bool; +} - // task_tokens = epoch * (taskref + 1) / total_priorities; - let task_tokens = epoch - .saturating_mul((t.priority as usize).saturating_add(1)) - .wrapping_div(total_priorities); +impl Returnable for bool { + fn should_return(&self) -> bool { + *self + } +} - t.tokens_remaining = task_tokens; - // debug!("assign_tokens(): CPU {} chose Task {:?}", cpu_id, &*t); - } +impl Returnable for Option { + fn should_return(&self) -> bool { + self.is_some() } } impl task::scheduler::Scheduler for Scheduler { fn next(&mut self) -> TaskRef { - self.try_next() - .or_else(|| { - self.assign_tokens(); - self.try_next() - }) - .unwrap_or_else(|| self.idle_task.clone()) + if self.active.is_empty() { + if !self.expired.is_empty() { + mem::swap(&mut self.active, &mut self.expired); + } else { + return self.idle_task.clone(); + } + } + self.active + .next(&mut self.expired) + .unwrap_or(self.idle_task.clone()) } fn add(&mut self, task: TaskRef) { - let priority_task_ref = EpochTaskRef::new(task); - self.queue.push_back(priority_task_ref); + let task = EpochTaskRef::new(task); + self.expired.push(task, DEFAULT_PRIORITY); } fn busyness(&self) -> usize { - self.queue.len() + self.active.len() + self.expired.len() } fn remove(&mut self, task: &TaskRef) -> bool { - let mut task_index = None; - for (i, t) in self.queue.iter().enumerate() { - if **t == *task { - task_index = Some(i); - break; - } - } - - if let Some(task_index) = task_index { - self.queue.remove(task_index); - true - } else { - false - } + self.apply(|run_queue| run_queue.remove(task)) } fn as_priority_scheduler(&mut self) -> Option<&mut dyn task::scheduler::PriorityScheduler> { @@ -157,14 +112,20 @@ impl task::scheduler::Scheduler for Scheduler { } fn drain(&mut self) -> Box + '_> { - Box::new(self.queue.drain(..).map(|epoch_task| epoch_task.task)) + let mut active = RunQueue::new(); + let mut expired = RunQueue::new(); + + mem::swap(&mut self.active, &mut active); + mem::swap(&mut self.expired, &mut expired); + + Box::new(active.drain().chain(expired.drain())) } fn tasks(&self) -> Vec { - self.queue + self.active .clone() - .into_iter() - .map(|epoch_task| epoch_task.task) + .drain() + .chain(self.expired.clone().drain()) .collect() } } @@ -172,30 +133,18 @@ impl task::scheduler::Scheduler for Scheduler { impl task::scheduler::PriorityScheduler for Scheduler { fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { let priority = core::cmp::min(priority, MAX_PRIORITY); - for epoch_task in self.queue.iter_mut() { - if epoch_task.task == *task { - epoch_task.priority = priority; - return true; - } - } - false + self.apply(|run_queue| run_queue.set_priority(task, priority)) } fn priority(&mut self, task: &TaskRef) -> Option { - for epoch_task in self.queue.iter() { - if epoch_task.task == *task { - return Some(epoch_task.priority); - } - } - None + self.apply(|run_queue| run_queue.priority(task)) } } #[derive(Debug, Clone)] struct EpochTaskRef { task: TaskRef, - priority: u8, - tokens_remaining: usize, + tokens: usize, } impl Deref for EpochTaskRef { @@ -216,8 +165,13 @@ impl EpochTaskRef { fn new(task: TaskRef) -> EpochTaskRef { EpochTaskRef { task, - priority: DEFAULT_PRIORITY, - tokens_remaining: INITIAL_TOKENS, + tokens: INITIAL_TOKENS, } } } + +impl From for TaskRef { + fn from(value: EpochTaskRef) -> Self { + value.task + } +} diff --git a/kernel/scheduler_epoch/src/queue.rs b/kernel/scheduler_epoch/src/queue.rs new file mode 100644 index 0000000000..4fdfb378b4 --- /dev/null +++ b/kernel/scheduler_epoch/src/queue.rs @@ -0,0 +1,178 @@ +use alloc::collections::VecDeque; + +use bit_set::BitSet; +use task::TaskRef; + +use crate::{EpochTaskRef, MAX_PRIORITY}; + +#[derive(Debug, Clone)] +pub(crate) struct RunQueue { + // TODO: Encode using MAX_PRIORITY + priorities: BitSet, + len: usize, + inner: [VecDeque; MAX_PRIORITY as usize], +} + +impl RunQueue { + pub(crate) const fn new() -> Self { + const INIT: VecDeque = VecDeque::new(); + + Self { + priorities: BitSet::new(), + len: 0, + inner: [INIT; MAX_PRIORITY as usize], + } + } + + pub(crate) const fn len(&self) -> usize { + self.len + } + + pub(crate) const fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub(crate) fn push(&mut self, task: EpochTaskRef, priority: u8) { + self.priorities.insert(priority); + self.inner[priority as usize].push_back(task); + self.len += 1; + } + + pub(crate) fn next(&mut self, expired: &mut Self) -> Option { + loop { + let top_index = self.top_index()?; + let top_queue = &mut self.inner[top_index]; + + // TODO: top_queue.len() == 1 optimisation + + let mut next_task = top_queue.pop_front().unwrap(); + + if !next_task.is_runnable() { + self.len -= 1; + expired.push(next_task.clone(), top_index as u8); + + if top_queue.is_empty() { + self.priorities.remove(top_index as u8); + } + } else if next_task.tokens <= 1 { + self.len -= 1; + expired.push(next_task.clone(), top_index as u8); + + if top_queue.is_empty() { + self.priorities.remove(top_index as u8); + } + + return Some(next_task.task); + } else { + next_task.tokens -= 1; + top_queue.push_back(next_task.clone()); + return Some(next_task.task); + } + } + } + + fn top_index(&self) -> Option { + self.priorities.max().map(|priority| priority as usize) + } + + pub(crate) fn remove(&mut self, task: &TaskRef) -> bool { + for i in self.priorities.iter() { + let queue = &mut self.inner[i]; + + for j in 0..queue.len() { + let element = &queue[j]; + + if **element == *task { + queue.remove(j); + self.len -= 1; + + if queue.is_empty() { + self.priorities.remove(i as u8); + } + + return true; + } + } + } + false + } + + /// Returns the priority of the given task. + pub(crate) fn priority(&self, task: &TaskRef) -> Option { + for i in self.priorities.iter() { + let queue = &self.inner[i]; + for t in queue { + if **t == *task { + return Some(i as u8); + } + } + } + None + } + + /// Sets the priority of the given task. + /// + /// Returns `true` if an action was performed i.e. if the task was in the + /// run queue. + pub(crate) fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { + for i in self.priorities.iter() { + let queue = &mut self.inner[i]; + + for j in 0..queue.len() { + let element = &queue[j]; + + if **element == *task { + let task = queue.remove(j).unwrap(); + self.len -= 1; + + if queue.is_empty() { + self.priorities.remove(i as u8); + } + + self.push(task, priority); + return true; + } + } + } + false + } + + pub(crate) fn drain(self) -> Drain { + Drain { inner: self } + } +} + +impl IntoIterator for RunQueue { + type Item = TaskRef; + + type IntoIter = Drain; + + fn into_iter(self) -> Self::IntoIter { + self.drain() + } +} + +pub(crate) struct Drain { + inner: RunQueue, +} + +impl Iterator for Drain { + type Item = TaskRef; + + fn next(&mut self) -> Option { + let top_index = self.inner.top_index()?; + + if top_index == 64 { + None + } else { + let top_queue = &mut self.inner.inner[top_index]; + + if top_queue.len() == 1 { + let priority = 64 - top_index; + self.inner.priorities.remove(priority as u8); + } + + Some(top_queue.pop_front().unwrap().into()) + } + } +} diff --git a/kernel/scheduler_epoch/src/queue/a.rs b/kernel/scheduler_epoch/src/queue/a.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kernel/scheduler_epoch/src/queue/priority.rs b/kernel/scheduler_epoch/src/queue/priority.rs new file mode 100644 index 0000000000..2b1423bb87 --- /dev/null +++ b/kernel/scheduler_epoch/src/queue/priority.rs @@ -0,0 +1,71 @@ +use alloc::collections::VecDeque; +use core::{intrinsics::unlikely, slice}; + +use crate::{queue::RunQueue, EpochTaskRef}; + +pub(crate) struct Iter<'a> { + inner: &'a RunQueue, + mask: u64, +} + +impl<'a> Iter<'a> { + pub(crate) fn new(run_queue: &'a RunQueue) -> Self { + Self { + inner: run_queue, + mask: u64::MAX, + } + } +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a VecDeque; + + fn next(&mut self) -> Option { + let next_index = (self.inner.priorities & self.mask).leading_zeros(); + + if next_index == 64 { + None + } else { + // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 + self.mask = u64::MAX.checked_shr(next_index + 1).unwrap_or(0); + Some(&self.inner.inner[next_index as usize]) + } + } +} + +pub(crate) struct IterMut<'a> { + current_index: i8, + priorities: u64, + inner: slice::IterMut<'a, VecDeque>, +} + +impl<'a> IterMut<'a> { + pub(crate) fn new(run_queue: &'a mut RunQueue) -> Self { + Self { + current_index: -1, + priorities: run_queue.priorities, + inner: run_queue.inner.iter_mut(), + } + } +} + +impl<'a> Iterator for IterMut<'a> { + type Item = &'a mut VecDeque; + + fn next(&mut self) -> Option { + // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 + if unlikely(self.current_index == 64) { + None + } else { + let mask = u64::MAX + .checked_shr((self.current_index + 1) as u32) + .unwrap_or(0); + let next_index = (self.priorities & mask).leading_zeros() as i8; + + let diff = next_index - self.current_index + 1; + self.current_index = next_index; + + self.inner.nth(diff as usize) + } + } +} diff --git a/libs/bit_set/Cargo.toml b/libs/bit_set/Cargo.toml new file mode 100644 index 0000000000..c0586429dc --- /dev/null +++ b/libs/bit_set/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "bit_set" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/libs/bit_set/src/iter.rs b/libs/bit_set/src/iter.rs new file mode 100644 index 0000000000..fd5d484d68 --- /dev/null +++ b/libs/bit_set/src/iter.rs @@ -0,0 +1,69 @@ +//! An iterator over a bit set. +//! +//! See [`Iter`] for more details. + +use core::intrinsics::unlikely; + +/// An iterator over a bit set. +pub struct Iter { + set: u64, + current_mask: u64, +} + +impl Iter { + pub(crate) const fn new(set: u64) -> Self { + Self { + set, + current_mask: u64::MAX, + } + } +} + +impl Iterator for Iter { + type Item = usize; + + fn next(&mut self) -> Option { + let next_index = (self.set & self.current_mask).trailing_zeros(); + + if unlikely(next_index == 64) { + None + } else { + // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 + self.current_mask = u64::MAX.checked_shl(next_index + 1).unwrap_or(0); + Some(next_index as usize) + } + } +} + +#[cfg(test)] +mod tests { + extern crate alloc; + + use alloc::vec::Vec; + + use crate::BitSet; + + #[test] + fn test_iter() { + let mut set = BitSet::new(); + set.insert(57); + set.insert(58); + set.insert(61); + set.insert(63); + assert_eq!(set.iter().collect::>(), [57, 58, 61, 63]); + + let mut set = BitSet::new(); + set.insert(0); + set.insert(8); + set.insert(16); + set.insert(24); + set.insert(32); + set.insert(40); + set.insert(48); + set.insert(56); + assert_eq!( + set.iter().collect::>(), + [0, 8, 16, 24, 32, 40, 48, 56] + ); + } +} diff --git a/libs/bit_set/src/lib.rs b/libs/bit_set/src/lib.rs new file mode 100644 index 0000000000..23bc4a0301 --- /dev/null +++ b/libs/bit_set/src/lib.rs @@ -0,0 +1,162 @@ +#![no_std] +#![feature(const_likely, core_intrinsics)] + +mod iter; + +use core::intrinsics::likely; + +pub use iter::Iter; + +/// A bit set backed by a [`u64`]. +/// +/// This is equivalent to a `HashSet` storing integers in the range `[0, +/// 64)`. +#[derive(Debug, Clone)] +pub struct BitSet { + inner: u64, +} + +impl BitSet { + /// Constructs a new, empty `BitSet`. + pub const fn new() -> Self { + Self { inner: 0 } + } + + /// Returns an iterator over the elements of the set. + #[must_use] + pub const fn iter(&self) -> Iter { + Iter::new(self.inner) + } + + /// Returns `true` if the set contains the given element. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + #[must_use] + pub const fn contains(&self, element: u8) -> bool { + assert!(element < 64); + self.inner & (1 << element) != 0 + } + + /// Adds an element to the set. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + pub fn insert(&mut self, element: u8) { + assert!(element < 64); + self.inner |= 1 << element; + } + + /// Removes an element from the set. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + pub fn remove(&mut self, element: u8) { + assert!(element < 64); + self.inner &= !(1 << element); + } + + /// Returns the smallest element in the set. + /// + /// Returns `None` if the set is empty. + #[must_use] + pub const fn min(&self) -> Option { + if likely(self.inner != 0) { + Some(self.inner.trailing_zeros() as u8) + } else { + None + } + } + + /// Returns the largest element in the set. + /// + /// Returns `None` if the set is empty. + #[must_use] + pub const fn max(&self) -> Option { + if likely(self.inner != 0) { + // self.inner.leading_zeros() <= 63 + Some(63 - self.inner.leading_zeros() as u8) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_contains() { + let mut set = BitSet::new(); + + for i in 0..64 { + assert!(!set.contains(i)); + } + + set.insert(3); + + for i in 0..64 { + if i != 3 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + + set.insert(0); + + for i in 0..64 { + if i != 0 && i != 3 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + + set.insert(63); + + for i in 0..64 { + if i != 0 && i != 3 && i != 63 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + } + + #[test] + fn test_remove() { + let mut set = BitSet::new(); + + set.insert(3); + set.insert(63); + set.remove(3); + + for i in 0..64 { + if i != 63 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + } + + #[test] + fn test_min_max() { + let mut set = BitSet::new(); + assert_eq!(set.min(), None); + assert_eq!(set.max(), None); + + set.insert(5); + assert_eq!(set.min(), Some(5)); + assert_eq!(set.max(), Some(5)); + + set.insert(3); + assert_eq!(set.min(), Some(3)); + assert_eq!(set.max(), Some(5)); + } +} From 50c045ff6611e114a39be92115a1be1bb893b18e Mon Sep 17 00:00:00 2001 From: Klimenty Tsoutsman Date: Wed, 27 Dec 2023 16:55:48 +1100 Subject: [PATCH 2/6] Add `bit_set` library Necessary for upcoming `epoch_scheduler` improvements. Signed-off-by: Klimenty Tsoutsman --- libs/bit_set/Cargo.toml | 6 ++ libs/bit_set/src/iter.rs | 67 ++++++++++++++++ libs/bit_set/src/lib.rs | 165 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 libs/bit_set/Cargo.toml create mode 100644 libs/bit_set/src/iter.rs create mode 100644 libs/bit_set/src/lib.rs diff --git a/libs/bit_set/Cargo.toml b/libs/bit_set/Cargo.toml new file mode 100644 index 0000000000..af52627a32 --- /dev/null +++ b/libs/bit_set/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "bit_set" +version = "0.1.0" +authors = ["Klim Tsoutsman "] +description = "A bit set storing integers less than 64" +edition = "2021" diff --git a/libs/bit_set/src/iter.rs b/libs/bit_set/src/iter.rs new file mode 100644 index 0000000000..135f36c3c2 --- /dev/null +++ b/libs/bit_set/src/iter.rs @@ -0,0 +1,67 @@ +use core::intrinsics::unlikely; + +/// An iterator over a [`BitSet`]. +/// +/// [`BitSet`]: crate::BitSet +pub struct Iter { + set: u64, + current_mask: u64, +} + +impl Iter { + pub(crate) const fn new(set: u64) -> Self { + Self { + set, + current_mask: u64::MAX, + } + } +} + +impl Iterator for Iter { + type Item = usize; + + fn next(&mut self) -> Option { + let next_index = (self.set & self.current_mask).trailing_zeros(); + + if unlikely(next_index == 64) { + None + } else { + // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 + self.current_mask = u64::MAX.checked_shl(next_index + 1).unwrap_or(0); + Some(next_index as usize) + } + } +} + +#[cfg(test)] +mod tests { + extern crate alloc; + + use alloc::vec::Vec; + + use crate::BitSet; + + #[test] + fn test_iter() { + let mut set = BitSet::new(); + set.insert(57); + set.insert(58); + set.insert(61); + set.insert(63); + assert_eq!(set.iter().collect::>(), [57, 58, 61, 63]); + + let mut set = BitSet::new(); + set.insert(0); + set.insert(8); + set.insert(16); + set.insert(24); + set.insert(32); + set.insert(40); + set.insert(48); + set.insert(56); + assert_eq!( + set.iter().collect::>(), + [0, 8, 16, 24, 32, 40, 48, 56] + ); + } +} diff --git a/libs/bit_set/src/lib.rs b/libs/bit_set/src/lib.rs new file mode 100644 index 0000000000..1724f46fc7 --- /dev/null +++ b/libs/bit_set/src/lib.rs @@ -0,0 +1,165 @@ +//! A bit set backed by a [`u64`]. +//! +//! See [`BitSet`] for more details. + +#![no_std] +#![feature(const_likely, core_intrinsics)] + +mod iter; + +use core::intrinsics::likely; + +pub use iter::Iter; + +/// A bit set backed by a [`u64`]. +/// +/// This is equivalent to a `HashSet` storing integers in the range `[0, +/// 64)`. +#[derive(Debug, Clone)] +pub struct BitSet { + inner: u64, +} + +impl BitSet { + /// Constructs a new, empty `BitSet`. + pub const fn new() -> Self { + Self { inner: 0 } + } + + /// Returns an iterator over the elements of the set. + pub const fn iter(&self) -> Iter { + Iter::new(self.inner) + } + + /// Returns `true` if the set contains the given element. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + #[must_use] + pub const fn contains(&self, element: u8) -> bool { + assert!(element < 64); + self.inner & (1 << element) != 0 + } + + /// Adds an element to the set. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + pub fn insert(&mut self, element: u8) { + assert!(element < 64); + self.inner |= 1 << element; + } + + /// Removes an element from the set. + /// + /// # Panics + /// + /// Panics if `element` is greater than 63. + pub fn remove(&mut self, element: u8) { + assert!(element < 64); + self.inner &= !(1 << element); + } + + /// Returns the smallest element in the set. + /// + /// Returns `None` if the set is empty. + #[must_use] + pub const fn min(&self) -> Option { + if likely(self.inner != 0) { + Some(self.inner.trailing_zeros() as u8) + } else { + None + } + } + + /// Returns the largest element in the set. + /// + /// Returns `None` if the set is empty. + #[must_use] + pub const fn max(&self) -> Option { + if likely(self.inner != 0) { + // self.inner.leading_zeros() <= 63 + Some(63 - self.inner.leading_zeros() as u8) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_contains() { + let mut set = BitSet::new(); + + for i in 0..64 { + assert!(!set.contains(i)); + } + + set.insert(3); + + for i in 0..64 { + if i != 3 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + + set.insert(0); + + for i in 0..64 { + if i != 0 && i != 3 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + + set.insert(63); + + for i in 0..64 { + if i != 0 && i != 3 && i != 63 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + } + + #[test] + fn test_remove() { + let mut set = BitSet::new(); + + set.insert(3); + set.insert(63); + set.remove(3); + + for i in 0..64 { + if i != 63 { + assert!(!set.contains(i)); + } else { + assert!(set.contains(i)); + } + } + } + + #[test] + fn test_min_max() { + let mut set = BitSet::new(); + assert_eq!(set.min(), None); + assert_eq!(set.max(), None); + + set.insert(5); + assert_eq!(set.min(), Some(5)); + assert_eq!(set.max(), Some(5)); + + set.insert(3); + assert_eq!(set.min(), Some(3)); + assert_eq!(set.max(), Some(5)); + } +} From 67b064bbae3dc9399bb4cd60814d2200459eb5c8 Mon Sep 17 00:00:00 2001 From: Klimenty Tsoutsman Date: Wed, 27 Dec 2023 19:02:09 +1100 Subject: [PATCH 3/6] Remove unnecessary files Signed-off-by: Klimenty Tsoutsman --- kernel/scheduler_epoch/src/queue/a.rs | 0 kernel/scheduler_epoch/src/queue/priority.rs | 71 -------------------- 2 files changed, 71 deletions(-) delete mode 100644 kernel/scheduler_epoch/src/queue/a.rs delete mode 100644 kernel/scheduler_epoch/src/queue/priority.rs diff --git a/kernel/scheduler_epoch/src/queue/a.rs b/kernel/scheduler_epoch/src/queue/a.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/kernel/scheduler_epoch/src/queue/priority.rs b/kernel/scheduler_epoch/src/queue/priority.rs deleted file mode 100644 index 2b1423bb87..0000000000 --- a/kernel/scheduler_epoch/src/queue/priority.rs +++ /dev/null @@ -1,71 +0,0 @@ -use alloc::collections::VecDeque; -use core::{intrinsics::unlikely, slice}; - -use crate::{queue::RunQueue, EpochTaskRef}; - -pub(crate) struct Iter<'a> { - inner: &'a RunQueue, - mask: u64, -} - -impl<'a> Iter<'a> { - pub(crate) fn new(run_queue: &'a RunQueue) -> Self { - Self { - inner: run_queue, - mask: u64::MAX, - } - } -} - -impl<'a> Iterator for Iter<'a> { - type Item = &'a VecDeque; - - fn next(&mut self) -> Option { - let next_index = (self.inner.priorities & self.mask).leading_zeros(); - - if next_index == 64 { - None - } else { - // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 - self.mask = u64::MAX.checked_shr(next_index + 1).unwrap_or(0); - Some(&self.inner.inner[next_index as usize]) - } - } -} - -pub(crate) struct IterMut<'a> { - current_index: i8, - priorities: u64, - inner: slice::IterMut<'a, VecDeque>, -} - -impl<'a> IterMut<'a> { - pub(crate) fn new(run_queue: &'a mut RunQueue) -> Self { - Self { - current_index: -1, - priorities: run_queue.priorities, - inner: run_queue.inner.iter_mut(), - } - } -} - -impl<'a> Iterator for IterMut<'a> { - type Item = &'a mut VecDeque; - - fn next(&mut self) -> Option { - // https://users.rust-lang.org/t/how-to-make-an-integer-with-n-bits-set-without-overflow/63078 - if unlikely(self.current_index == 64) { - None - } else { - let mask = u64::MAX - .checked_shr((self.current_index + 1) as u32) - .unwrap_or(0); - let next_index = (self.priorities & mask).leading_zeros() as i8; - - let diff = next_index - self.current_index + 1; - self.current_index = next_index; - - self.inner.nth(diff as usize) - } - } -} From 6ed96c3611a063ec5822f081f771b11477fc4cb2 Mon Sep 17 00:00:00 2001 From: Klim Tsoutsman Date: Thu, 4 Jan 2024 20:18:33 +1100 Subject: [PATCH 4/6] Temp Signed-off-by: Klim Tsoutsman --- Cargo.lock | 1 + kernel/scheduler_epoch/Cargo.toml | 1 + kernel/scheduler_epoch/src/lib.rs | 56 +++++++++++++++++++++++++++-- kernel/scheduler_epoch/src/queue.rs | 14 ++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f647bca32..47512af1e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,6 +3146,7 @@ name = "scheduler_epoch" version = "0.1.0" dependencies = [ "bit_set", + "kernel_config", "log", "spin 0.9.4", "task", diff --git a/kernel/scheduler_epoch/Cargo.toml b/kernel/scheduler_epoch/Cargo.toml index 3a877a6c23..63d7c675ae 100644 --- a/kernel/scheduler_epoch/Cargo.toml +++ b/kernel/scheduler_epoch/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] bit_set = { path = "../../libs/bit_set" } +kernel_config = { path = "../kernel_config" } log = "0.4.8" spin = "0.9.4" task = { path = "../task" } diff --git a/kernel/scheduler_epoch/src/lib.rs b/kernel/scheduler_epoch/src/lib.rs index a25ed46003..40902510b9 100644 --- a/kernel/scheduler_epoch/src/lib.rs +++ b/kernel/scheduler_epoch/src/lib.rs @@ -3,6 +3,8 @@ //! The implementation is based on the [`O(1)` Linux //! scheduler][linux-scheduler]. //! +//! The scheduler is comprised of two run queues: an . +//! //! [linux-scheduler]: https://litux.nl/mirror/kerneldevelopment/0672327201/ch04lev1sec2.html #![no_std] @@ -16,6 +18,7 @@ use alloc::{boxed::Box, vec::Vec}; use core::{ mem, ops::{Deref, DerefMut}, + time::Duration, }; use task::TaskRef; @@ -24,13 +27,20 @@ use crate::queue::RunQueue; const MAX_PRIORITY: u8 = 63; const DEFAULT_PRIORITY: u8 = 20; -const INITIAL_TOKENS: usize = 0; -/// An instance of an epoch scheduler, typically one per CPU. +/// The minimum amount of time for every runnable task to run. +/// +/// This is not strictly adhered to when the tasks are run +const TARGET_LATENCY: Duration = Duration::from_millis(15); + +/// An epoch scheduler. +/// +/// See crate-level docs for more information. pub struct Scheduler { idle_task: TaskRef, active: RunQueue, expired: RunQueue, + total_weight: usize, } impl Scheduler { @@ -40,6 +50,8 @@ impl Scheduler { idle_task, active: RunQueue::new(), expired: RunQueue::new(), + // TODO: 0 or 1 + total_weight: 0, } } @@ -81,6 +93,7 @@ impl Returnable for Option { } impl task::scheduler::Scheduler for Scheduler { + #[inline] fn next(&mut self) -> TaskRef { if self.active.is_empty() { if !self.expired.is_empty() { @@ -94,23 +107,29 @@ impl task::scheduler::Scheduler for Scheduler { .unwrap_or(self.idle_task.clone()) } + #[inline] fn add(&mut self, task: TaskRef) { - let task = EpochTaskRef::new(task); + let (task, weight) = EpochTaskRef::new(task, self.total_weight); + self.total_weight += weight; self.expired.push(task, DEFAULT_PRIORITY); } + #[inline] fn busyness(&self) -> usize { self.active.len() + self.expired.len() } + #[inline] fn remove(&mut self, task: &TaskRef) -> bool { self.apply(|run_queue| run_queue.remove(task)) } + #[inline] fn as_priority_scheduler(&mut self) -> Option<&mut dyn task::scheduler::PriorityScheduler> { Some(self) } + #[inline] fn drain(&mut self) -> Box + '_> { let mut active = RunQueue::new(); let mut expired = RunQueue::new(); @@ -121,6 +140,7 @@ impl task::scheduler::Scheduler for Scheduler { Box::new(active.drain().chain(expired.drain())) } + #[inline] fn tasks(&self) -> Vec { self.active .clone() @@ -131,11 +151,13 @@ impl task::scheduler::Scheduler for Scheduler { } impl task::scheduler::PriorityScheduler for Scheduler { + #[inline] fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { let priority = core::cmp::min(priority, MAX_PRIORITY); self.apply(|run_queue| run_queue.set_priority(task, priority)) } + #[inline] fn priority(&mut self, task: &TaskRef) -> Option { self.apply(|run_queue| run_queue.priority(task)) } @@ -147,21 +169,48 @@ struct EpochTaskRef { tokens: usize, } +impl EpochTaskRef { + #[must_use] + pub(crate) fn new(task: TaskRef, config: TaskConfiguration) -> (Self, usize) { + const NUM_TOKENS: usize = + TARGET_LATENCY / kernel_config::time::CONFIG_TIMESLICE_PERIOD_MICROSECONDS; + + // TODO + let weight = config.priority + 1; + + ( + Self { + task, + tokens: core::cmp::max(NUM_TOKENS * weight / config.total_weight, 1), + }, + weight, + ) + } +} + +pub(crate) struct TaskConfiguration { + pub(crate) priority: usize, + pub(crate) total_weight: usize, +} + impl Deref for EpochTaskRef { type Target = TaskRef; + #[inline] fn deref(&self) -> &TaskRef { &self.task } } impl DerefMut for EpochTaskRef { + #[inline] fn deref_mut(&mut self) -> &mut TaskRef { &mut self.task } } impl EpochTaskRef { + #[inline] fn new(task: TaskRef) -> EpochTaskRef { EpochTaskRef { task, @@ -171,6 +220,7 @@ impl EpochTaskRef { } impl From for TaskRef { + #[inline] fn from(value: EpochTaskRef) -> Self { value.task } diff --git a/kernel/scheduler_epoch/src/queue.rs b/kernel/scheduler_epoch/src/queue.rs index 4fdfb378b4..90fe309785 100644 --- a/kernel/scheduler_epoch/src/queue.rs +++ b/kernel/scheduler_epoch/src/queue.rs @@ -5,6 +5,9 @@ use task::TaskRef; use crate::{EpochTaskRef, MAX_PRIORITY}; +/// A singular run queue. +/// +/// The scheduler contains two of these: an active one, and an expired one. #[derive(Debug, Clone)] pub(crate) struct RunQueue { // TODO: Encode using MAX_PRIORITY @@ -14,6 +17,7 @@ pub(crate) struct RunQueue { } impl RunQueue { + #[inline] pub(crate) const fn new() -> Self { const INIT: VecDeque = VecDeque::new(); @@ -24,20 +28,25 @@ impl RunQueue { } } + #[inline] pub(crate) const fn len(&self) -> usize { + debug_assert_eq!(self.inner.iter().map(|queue| queue.len()).sum(), self.len); self.len } + #[inline] pub(crate) const fn is_empty(&self) -> bool { self.len() == 0 } + #[inline] pub(crate) fn push(&mut self, task: EpochTaskRef, priority: u8) { self.priorities.insert(priority); self.inner[priority as usize].push_back(task); self.len += 1; } + #[inline] pub(crate) fn next(&mut self, expired: &mut Self) -> Option { loop { let top_index = self.top_index()?; @@ -71,10 +80,12 @@ impl RunQueue { } } + #[inline] fn top_index(&self) -> Option { self.priorities.max().map(|priority| priority as usize) } + #[inline] pub(crate) fn remove(&mut self, task: &TaskRef) -> bool { for i in self.priorities.iter() { let queue = &mut self.inner[i]; @@ -98,6 +109,7 @@ impl RunQueue { } /// Returns the priority of the given task. + #[inline] pub(crate) fn priority(&self, task: &TaskRef) -> Option { for i in self.priorities.iter() { let queue = &self.inner[i]; @@ -114,6 +126,7 @@ impl RunQueue { /// /// Returns `true` if an action was performed i.e. if the task was in the /// run queue. + #[inline] pub(crate) fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { for i in self.priorities.iter() { let queue = &mut self.inner[i]; @@ -137,6 +150,7 @@ impl RunQueue { false } + #[inline] pub(crate) fn drain(self) -> Drain { Drain { inner: self } } From abc169f59a3806cf3fddd42c329dd979c82a38d0 Mon Sep 17 00:00:00 2001 From: Klim Tsoutsman Date: Thu, 4 Jan 2024 22:52:17 +1100 Subject: [PATCH 5/6] Properly implement rehydration Signed-off-by: Klim Tsoutsman --- kernel/scheduler_epoch/src/lib.rs | 46 +++++----- kernel/scheduler_epoch/src/queue.rs | 131 +++++++++++++++++++--------- 2 files changed, 117 insertions(+), 60 deletions(-) diff --git a/kernel/scheduler_epoch/src/lib.rs b/kernel/scheduler_epoch/src/lib.rs index 40902510b9..0fe36ea4b3 100644 --- a/kernel/scheduler_epoch/src/lib.rs +++ b/kernel/scheduler_epoch/src/lib.rs @@ -5,6 +5,9 @@ //! //! The scheduler is comprised of two run queues: an . //! +//! Note that our implementation is not constant-time since we store +//! non-runnable tasks on the run queue. +//! //! [linux-scheduler]: https://litux.nl/mirror/kerneldevelopment/0672327201/ch04lev1sec2.html #![no_std] @@ -103,13 +106,19 @@ impl task::scheduler::Scheduler for Scheduler { } } self.active - .next(&mut self.expired) + .next(&mut self.expired, self.total_weight) .unwrap_or(self.idle_task.clone()) } #[inline] fn add(&mut self, task: TaskRef) { - let (task, weight) = EpochTaskRef::new(task, self.total_weight); + let (task, weight) = EpochTaskRef::new( + task, + TaskConfiguration { + priority: DEFAULT_PRIORITY as usize, + total_weight: self.total_weight, + }, + ); self.total_weight += weight; self.expired.push(task, DEFAULT_PRIORITY); } @@ -170,21 +179,26 @@ struct EpochTaskRef { } impl EpochTaskRef { + /// Creates a new task. + /// + /// Returns the task and the weight of the task. #[must_use] pub(crate) fn new(task: TaskRef, config: TaskConfiguration) -> (Self, usize) { - const NUM_TOKENS: usize = - TARGET_LATENCY / kernel_config::time::CONFIG_TIMESLICE_PERIOD_MICROSECONDS; + let mut task = Self { task, tokens: 0 }; + let weight = task.recalculate_tokens(config); + (task, weight) + } + + #[inline] + pub(crate) fn recalculate_tokens(&mut self, config: TaskConfiguration) -> usize { + const TOTAL_TOKENS: usize = TARGET_LATENCY.as_micros() as usize + / kernel_config::time::CONFIG_TIMESLICE_PERIOD_MICROSECONDS as usize; // TODO let weight = config.priority + 1; + self.tokens = core::cmp::max(TOTAL_TOKENS * weight / config.total_weight, 1); - ( - Self { - task, - tokens: core::cmp::max(NUM_TOKENS * weight / config.total_weight, 1), - }, - weight, - ) + weight } } @@ -209,16 +223,6 @@ impl DerefMut for EpochTaskRef { } } -impl EpochTaskRef { - #[inline] - fn new(task: TaskRef) -> EpochTaskRef { - EpochTaskRef { - task, - tokens: INITIAL_TOKENS, - } - } -} - impl From for TaskRef { #[inline] fn from(value: EpochTaskRef) -> Self { diff --git a/kernel/scheduler_epoch/src/queue.rs b/kernel/scheduler_epoch/src/queue.rs index 90fe309785..042089b7eb 100644 --- a/kernel/scheduler_epoch/src/queue.rs +++ b/kernel/scheduler_epoch/src/queue.rs @@ -3,7 +3,7 @@ use alloc::collections::VecDeque; use bit_set::BitSet; use task::TaskRef; -use crate::{EpochTaskRef, MAX_PRIORITY}; +use crate::{EpochTaskRef, TaskConfiguration, MAX_PRIORITY}; /// A singular run queue. /// @@ -29,13 +29,16 @@ impl RunQueue { } #[inline] - pub(crate) const fn len(&self) -> usize { - debug_assert_eq!(self.inner.iter().map(|queue| queue.len()).sum(), self.len); + pub(crate) fn len(&self) -> usize { + debug_assert_eq!( + self.inner.iter().map(|queue| queue.len()).sum::(), + self.len + ); self.len } #[inline] - pub(crate) const fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.len() == 0 } @@ -47,37 +50,93 @@ impl RunQueue { } #[inline] - pub(crate) fn next(&mut self, expired: &mut Self) -> Option { - loop { - let top_index = self.top_index()?; - let top_queue = &mut self.inner[top_index]; + pub(crate) fn next(&mut self, expired: &mut Self, total_weight: usize) -> Option { + let mut priorities = self.priorities.clone(); + + let mut top_index = priorities.max()?; + // TODO: top_queue.len() == 1 optimisation + let mut top_queue = &mut self.inner[top_index as usize]; + let mut next_task = top_queue.front().unwrap(); + + if !next_task.is_runnable() { + // TODO: This incredibly convoluted code is necessary because we store + // non-runnable tasks on the run queue. + + // Iterate through the queue to find the next runnable task and bring it to the + // front of its respective run queue. + + let mut vec_index = 0; + + while !next_task.is_runnable() { + vec_index += 1; + + if vec_index + 1 == top_queue.len() { + priorities.remove(top_index); + top_index = match priorities.max() { + Some(top) => top, + None => { + // There are no runnable tasks on the run queue. We + // must transfer all the tasks to the expired run + // queue and return None. + + let mut priorities = self.priorities.clone(); + + while let Some(top_index) = priorities.max() { + let top_queue = &mut self.inner[top_index as usize]; + + while let Some(mut task) = top_queue.pop_front() { + task.recalculate_tokens(TaskConfiguration { + priority: top_index as usize, + total_weight, + }); + expired.push(task, top_index); + } + + priorities.remove(top_index); + } + + return None; + } + }; + vec_index = 0; + } - // TODO: top_queue.len() == 1 optimisation + top_queue = &mut self.inner[top_index as usize]; + next_task = &top_queue[vec_index]; + } - let mut next_task = top_queue.pop_front().unwrap(); + for _ in 0..vec_index { + let task = top_queue.pop_front().unwrap(); + top_queue.push_back(task); + } + } - if !next_task.is_runnable() { - self.len -= 1; - expired.push(next_task.clone(), top_index as u8); + let queue = &mut self.inner[top_index as usize]; + let next_task = queue.front().unwrap(); - if top_queue.is_empty() { - self.priorities.remove(top_index as u8); - } - } else if next_task.tokens <= 1 { - self.len -= 1; - expired.push(next_task.clone(), top_index as u8); + Some(if next_task.tokens <= 1 { + let mut next_task = queue.pop_front().unwrap(); + self.len -= 1; - if top_queue.is_empty() { - self.priorities.remove(top_index as u8); - } + next_task.recalculate_tokens(TaskConfiguration { + priority: top_index as usize, + total_weight, + }); + expired.push(next_task.clone(), top_index); - return Some(next_task.task); - } else { - next_task.tokens -= 1; - top_queue.push_back(next_task.clone()); - return Some(next_task.task); + if queue.is_empty() { + self.priorities.remove(top_index); } - } + + next_task.clone().task + } else { + let mut next_task = queue.pop_front().unwrap(); + + next_task.tokens -= 1; + queue.push_back(next_task.clone()); + + next_task.task + }) } #[inline] @@ -175,18 +234,12 @@ impl Iterator for Drain { fn next(&mut self) -> Option { let top_index = self.inner.top_index()?; + let top_queue = &mut self.inner.inner[top_index]; - if top_index == 64 { - None - } else { - let top_queue = &mut self.inner.inner[top_index]; - - if top_queue.len() == 1 { - let priority = 64 - top_index; - self.inner.priorities.remove(priority as u8); - } - - Some(top_queue.pop_front().unwrap().into()) + if top_queue.len() == 1 { + self.inner.priorities.remove(top_index as u8); } + + Some(top_queue.pop_front().unwrap().into()) } } From 464345274b919308497c361d80b779397d160fa7 Mon Sep 17 00:00:00 2001 From: Klim Tsoutsman Date: Sat, 6 Jan 2024 12:43:01 +1100 Subject: [PATCH 6/6] Complete implementation Signed-off-by: Klim Tsoutsman --- kernel/scheduler_epoch/src/lib.rs | 43 +++++++++++++++++++---------- kernel/scheduler_epoch/src/queue.rs | 30 ++++++++++++-------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/kernel/scheduler_epoch/src/lib.rs b/kernel/scheduler_epoch/src/lib.rs index 0fe36ea4b3..456691e0ee 100644 --- a/kernel/scheduler_epoch/src/lib.rs +++ b/kernel/scheduler_epoch/src/lib.rs @@ -58,9 +58,9 @@ impl Scheduler { } } - fn apply(&mut self, f: F) -> R + fn apply(&mut self, mut f: F) -> R where - F: Fn(&mut RunQueue) -> R, + F: FnMut(&mut RunQueue) -> R, R: Returnable, { let (first, second) = if self.active.len() >= self.expired.len() { @@ -112,14 +112,16 @@ impl task::scheduler::Scheduler for Scheduler { #[inline] fn add(&mut self, task: TaskRef) { - let (task, weight) = EpochTaskRef::new( + let weight = weight(DEFAULT_PRIORITY); + self.total_weight += weight; + + let task = EpochTaskRef::new( task, TaskConfiguration { - priority: DEFAULT_PRIORITY as usize, + weight, total_weight: self.total_weight, }, ); - self.total_weight += weight; self.expired.push(task, DEFAULT_PRIORITY); } @@ -130,7 +132,13 @@ impl task::scheduler::Scheduler for Scheduler { #[inline] fn remove(&mut self, task: &TaskRef) -> bool { - self.apply(|run_queue| run_queue.remove(task)) + match self.apply(|run_queue| run_queue.remove(task)) { + Some(weight) => { + self.total_weight -= weight; + true + } + None => false, + } } #[inline] @@ -145,6 +153,7 @@ impl task::scheduler::Scheduler for Scheduler { mem::swap(&mut self.active, &mut active); mem::swap(&mut self.expired, &mut expired); + self.total_weight = 0; Box::new(active.drain().chain(expired.drain())) } @@ -172,6 +181,11 @@ impl task::scheduler::PriorityScheduler for Scheduler { } } +#[inline] +fn weight(priority: u8) -> usize { + priority as usize + 1 +} + #[derive(Debug, Clone)] struct EpochTaskRef { task: TaskRef, @@ -183,27 +197,26 @@ impl EpochTaskRef { /// /// Returns the task and the weight of the task. #[must_use] - pub(crate) fn new(task: TaskRef, config: TaskConfiguration) -> (Self, usize) { + pub(crate) fn new(task: TaskRef, config: TaskConfiguration) -> Self { let mut task = Self { task, tokens: 0 }; - let weight = task.recalculate_tokens(config); - (task, weight) + task.recalculate_tokens(config); + task } #[inline] - pub(crate) fn recalculate_tokens(&mut self, config: TaskConfiguration) -> usize { + pub(crate) fn recalculate_tokens(&mut self, config: TaskConfiguration) { const TOTAL_TOKENS: usize = TARGET_LATENCY.as_micros() as usize / kernel_config::time::CONFIG_TIMESLICE_PERIOD_MICROSECONDS as usize; // TODO - let weight = config.priority + 1; - self.tokens = core::cmp::max(TOTAL_TOKENS * weight / config.total_weight, 1); - - weight + self.tokens = core::cmp::max(TOTAL_TOKENS * config.weight / config.total_weight, 1); } } pub(crate) struct TaskConfiguration { - pub(crate) priority: usize, + /// The weight of the task. + pub(crate) weight: usize, + /// The sum of the weights of all tasks on the run queue. pub(crate) total_weight: usize, } diff --git a/kernel/scheduler_epoch/src/queue.rs b/kernel/scheduler_epoch/src/queue.rs index 042089b7eb..f55f11192e 100644 --- a/kernel/scheduler_epoch/src/queue.rs +++ b/kernel/scheduler_epoch/src/queue.rs @@ -3,7 +3,7 @@ use alloc::collections::VecDeque; use bit_set::BitSet; use task::TaskRef; -use crate::{EpochTaskRef, TaskConfiguration, MAX_PRIORITY}; +use crate::{weight, EpochTaskRef, TaskConfiguration, MAX_PRIORITY}; /// A singular run queue. /// @@ -68,7 +68,7 @@ impl RunQueue { let mut vec_index = 0; while !next_task.is_runnable() { - vec_index += 1; + assert!(!top_queue.is_empty()); if vec_index + 1 == top_queue.len() { priorities.remove(top_index); @@ -86,7 +86,7 @@ impl RunQueue { while let Some(mut task) = top_queue.pop_front() { task.recalculate_tokens(TaskConfiguration { - priority: top_index as usize, + weight: weight(top_index), total_weight, }); expired.push(task, top_index); @@ -95,13 +95,17 @@ impl RunQueue { priorities.remove(top_index); } + self.priorities = BitSet::new(); + self.len = 0; + return None; } }; + top_queue = &mut self.inner[top_index as usize]; vec_index = 0; } - top_queue = &mut self.inner[top_index as usize]; + vec_index += 1; next_task = &top_queue[vec_index]; } @@ -119,7 +123,7 @@ impl RunQueue { self.len -= 1; next_task.recalculate_tokens(TaskConfiguration { - priority: top_index as usize, + weight: weight(top_index), total_weight, }); expired.push(next_task.clone(), top_index); @@ -145,9 +149,9 @@ impl RunQueue { } #[inline] - pub(crate) fn remove(&mut self, task: &TaskRef) -> bool { - for i in self.priorities.iter() { - let queue = &mut self.inner[i]; + pub(crate) fn remove(&mut self, task: &TaskRef) -> Option { + for priority in self.priorities.iter() { + let queue = &mut self.inner[priority]; for j in 0..queue.len() { let element = &queue[j]; @@ -157,14 +161,14 @@ impl RunQueue { self.len -= 1; if queue.is_empty() { - self.priorities.remove(i as u8); + self.priorities.remove(priority as u8); } - return true; + return Some(weight(priority as u8)); } } } - false + None } /// Returns the priority of the given task. @@ -187,6 +191,7 @@ impl RunQueue { /// run queue. #[inline] pub(crate) fn set_priority(&mut self, task: &TaskRef, priority: u8) -> bool { + // FIXME: Recalculate weights. for i in self.priorities.iter() { let queue = &mut self.inner[i]; @@ -206,7 +211,8 @@ impl RunQueue { } } } - false + // false + todo!(); } #[inline]