diff --git a/rjit_c.rb b/rjit_c.rb index 802c7fb7fa0199..c9595df6369e3a 100644 --- a/rjit_c.rb +++ b/rjit_c.rb @@ -1065,7 +1065,6 @@ def C.rb_control_frame_t ep: [CType::Pointer.new { self.VALUE }, Primitive.cexpr!("OFFSETOF((*((struct rb_control_frame_struct *)NULL)), ep)")], block_code: [CType::Immediate.parse("void *"), Primitive.cexpr!("OFFSETOF((*((struct rb_control_frame_struct *)NULL)), block_code)")], jit_return: [CType::Pointer.new { CType::Immediate.parse("void") }, Primitive.cexpr!("OFFSETOF((*((struct rb_control_frame_struct *)NULL)), jit_return)")], - jit_frame: [CType::Pointer.new { self.rb_jit_frame_t }, Primitive.cexpr!("OFFSETOF((*((struct rb_control_frame_struct *)NULL)), jit_frame)")], ) end diff --git a/vm.c b/vm.c index 41115a723b3412..eecc2908d923ab 100644 --- a/vm.c +++ b/vm.c @@ -198,7 +198,7 @@ VM_CAPTURED_BLOCK_TO_CFP(const struct rb_captured_block *captured) { rb_control_frame_t *cfp = ((rb_control_frame_t *)((VALUE *)(captured) - 3)); VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp)); - VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 8 + VM_DEBUG_BP_CHECK ? 1 : 0); + VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 7 + VM_DEBUG_BP_CHECK ? 1 : 0); return cfp; } diff --git a/vm_core.h b/vm_core.h index cb1d367170b497..abfaf74dd501f9 100644 --- a/vm_core.h +++ b/vm_core.h @@ -826,22 +826,25 @@ typedef struct rb_control_frame_struct { const VALUE *ep; // cfp[4] / block[1] const void *block_code; // cfp[5] / block[2] -- iseq, ifunc, or forwarded block handler void *jit_return; // cfp[6] -- return address for JIT code - rb_jit_frame_t *jit_frame; // cfp[7] #if VM_DEBUG_BP_CHECK VALUE *bp_check; // cfp[8] #endif } rb_control_frame_t; -#define ASSERT_FRAME_MATERIALIZED(cfp) RUBY_ASSERT(! (cfp)->jit_frame) +#define JIT_FRAME_P(cfp) (((VALUE)(cfp)->_pc) & 1) -#define JIT_FRAME_CFRAME_P(cfp) (VM_FRAME_MAGIC_CFUNC == ((cfp)->jit_frame->flags & VM_FRAME_MAGIC_MASK)) +#define JIT_FRAME(cfp) ((const rb_jit_frame_t *)(((VALUE)(cfp)->_pc) - 1)) -#define CFP_PC(cfp) ((cfp)->jit_frame ? (cfp)->jit_frame->pc : (cfp)->_pc) +#define ASSERT_FRAME_MATERIALIZED(cfp) RUBY_ASSERT(! JIT_FRAME_P(cfp)) + +#define JIT_FRAME_CFRAME_P(cfp) (VM_FRAME_MAGIC_CFUNC == (JIT_FRAME(cfp)->flags & VM_FRAME_MAGIC_MASK)) + +#define CFP_PC(cfp) (JIT_FRAME_P(cfp) ? JIT_FRAME(cfp)->pc : (cfp)->_pc) VALUE *rb_vm_jit_frame_sp(const rb_control_frame_t *cfp); -#define CFP_SP(cfp) (((cfp)->jit_frame) ? rb_vm_jit_frame_sp(cfp) : (cfp)->_sp) +#define CFP_SP(cfp) (JIT_FRAME_P(cfp) ? rb_vm_jit_frame_sp(cfp) : (cfp)->_sp) -#define CFP_ISEQ(cfp) (((cfp)->jit_frame && JIT_FRAME_CFRAME_P(cfp)) ? 0 : (cfp)->_iseq) +#define CFP_ISEQ(cfp) ((JIT_FRAME_P(cfp) && JIT_FRAME_CFRAME_P(cfp)) ? 0 : (cfp)->_iseq) extern const rb_data_type_t ruby_threadptr_data_type; @@ -1334,11 +1337,12 @@ VM_FRAME_BMETHOD_P(const rb_control_frame_t *cfp) static inline void rb_vm_cfp_materialize(rb_control_frame_t *cfp) { - if (cfp->jit_frame) { - cfp->_pc = CFP_PC(cfp); + if (JIT_FRAME_P(cfp)) { cfp->_sp = CFP_SP(cfp); - if (cfp->jit_frame->flags & VM_FRAME_FLAG_CFRAME) cfp->_iseq = 0; - cfp->jit_frame = NULL; + if (JIT_FRAME(cfp)->flags & VM_FRAME_FLAG_CFRAME) cfp->_iseq = 0; + // This needs to be the last thought as setting the actual PC + // makes this no longer a jit frame + cfp->_pc = CFP_PC(cfp); } } diff --git a/vm_insnhelper.c b/vm_insnhelper.c index f8c8456cc47ed4..ae82a1ab96f3f0 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -383,7 +383,6 @@ vm_push_frame(rb_execution_context_t *ec, .ep = sp - 1, .block_code = NULL, .jit_return = NULL, - .jit_frame = NULL, #if VM_DEBUG_BP_CHECK .bp_check = sp, #endif @@ -2504,7 +2503,7 @@ rb_vm_jit_frame_sp(const rb_control_frame_t *cfp) { // Sum offsets of all jit frames ptrdiff_t tally = 0; - while (cfp->jit_frame) { + while (JIT_FRAME_P(cfp)) { if (CFP_ISEQ(cfp)) { tally += ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE; if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { @@ -2512,7 +2511,7 @@ rb_vm_jit_frame_sp(const rb_control_frame_t *cfp) tally += 1; } } - tally += cfp->jit_frame->sp_offset; + tally += JIT_FRAME(cfp)->sp_offset; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } return cfp->_sp + tally; diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 1134c1a185c537..e6ae5db3b51722 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -274,7 +274,6 @@ fn jit_save_pc(jit: &JITState, asm: &mut Assembler) { asm_comment!(asm, "save PC to CFP"); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(ptr as *const u8)); - //asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_FRAME), 0.into()); } /// For calling routines in the middle of a YARV instruction that will @@ -322,7 +321,7 @@ fn jit_bump_progress(jit: &mut JITState, asm: &mut Assembler) { sp_offset: asm.ctx.get_stack_size().into(), flags: VALUE(0), }.move_to_heap(); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_FRAME), Opnd::const_ptr(jit_frame as _)); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), (jit_frame as usize + 1).into()); } /// Record the current codeblock write position for rewriting into a jump into @@ -462,12 +461,6 @@ fn gen_exit(exit_pc: *mut VALUE, asm: &mut Assembler) { // Spill stack temps before returning to the interpreter asm.spill_temps(); - // TODO: Call the materialize function - asm.mov( - Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_FRAME), - Opnd::const_ptr(0 as *const u8) - ); - // Generate the code to exit to the interpreters // Write the adjusted SP back into the CFP if true /* REG_SP and cfp->sp aren't in sync anymore with outlining */ { @@ -5298,9 +5291,7 @@ fn gen_push_frame( sp_offset: VM_ENV_DATA_SIZE as i32 + frame.cfunc_args_and_self, flags: VALUE(frame.frame_type.as_usize()), }.move_to_heap(); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_FRAME), Opnd::const_ptr(cframe_jit_frame as _)); - } else { - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_FRAME), 0.into()); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), (cframe_jit_frame as usize + 1).into()); } // Spill stack temps to let the callee use them (must be done before changing SP) @@ -6529,16 +6520,15 @@ fn gen_send_iseq( let sp_offset = (argc as isize) + if captured_self { 0 } else { 1 }; // Store the updated SP on the current frame (pop arguments and receiver) + // This helps with reconstructing interpreter state when we need to side exit. + // Normal JIT-to-JIT return also read from this SP field. asm_comment!(asm, "store caller sp"); let caller_sp = asm.lea(asm.ctx.sp_opnd((SIZEOF_VALUE as isize) * -sp_offset)); asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), caller_sp); - // Store the next PC in the current frame + // Store the next PC in the current frame. + // Becuse the PC is not tagged, this also makes the caller frame not a JIT frame. jit_save_pc(jit, asm); - // TODO(outline): ^^^^ this and the sp motion above should become a jit_progress_bump() later - // but it's hard because gen_leave() reads from cfp->_sp directly to restore - // REG_SP when callee pops - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_FRAME), 0.into()); // Adjust the callee's stack pointer let offs = (SIZEOF_VALUE as isize) * ( diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index 5577b12dfd8325..832811650d9be4 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -718,8 +718,7 @@ mod manual_defs { pub const RUBY_OFFSET_CFP_EP: i32 = 32; pub const RUBY_OFFSET_CFP_BLOCK_CODE: i32 = 40; pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48; - pub const RUBY_OFFSET_CFP_JIT_FRAME: i32 = 56; - pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 64; + pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56; // Constants from rb_execution_context_t vm_core.h pub const RUBY_OFFSET_EC_CFP: i32 = 16;