Skip to content

Commit

Permalink
Refactor scheduler step functions
Browse files Browse the repository at this point in the history
  • Loading branch information
mmoskal committed Jan 16, 2024
1 parent f247d54 commit c050aef
Showing 1 changed file with 6 additions and 5 deletions.
11 changes: 6 additions & 5 deletions rllm/src/paged/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ impl Scheduler {
.sum()
}

fn step_start_waiting(&mut self, outputs: &mut SchedulerOutputs) {
fn step_prompts(&mut self, outputs: &mut SchedulerOutputs) {
log::trace!("step_start_waiting ({} seqs)", self.q_len(Queue::Waiting));
self.sort_by_priority(Queue::Waiting);

Expand Down Expand Up @@ -295,7 +295,9 @@ impl Scheduler {
});
}

fn step_preempt(&mut self, outputs: &mut SchedulerOutputs) -> bool {
/// Move sequences from OnGpu queue to outputs.next_seq_groups or
/// to Swapped/Waiting queues (preemption).
fn step_generation(&mut self, outputs: &mut SchedulerOutputs) -> bool {
let mut did_preempt = false;
self.sort_by_priority(Queue::OnGpu);

Expand Down Expand Up @@ -398,12 +400,11 @@ impl Scheduler {
self.step_drop_finished(&mut outputs);

if self.q_len(Queue::Swapped) == 0 {
self.step_start_waiting(&mut outputs);
self.step_prompts(&mut outputs);
}

if outputs.next_seq_groups.is_empty() {
// Preemption logic
let did_preempt = self.step_preempt(&mut outputs);
let did_preempt = self.step_generation(&mut outputs);

// Swap in logic for swapped sequences
if !did_preempt {
Expand Down

0 comments on commit c050aef

Please sign in to comment.