From cd8d20cd1fbcf9bf9d438b306beb65b2417fcc04 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 8 Aug 2023 16:06:22 -0700 Subject: YJIT: Compile exception handlers (#8171) Co-authored-by: Maxime Chevalier-Boisvert --- yjit/bindgen/src/main.rs | 2 + yjit/src/codegen.rs | 96 +++++++++++++++++++++++++++++++++++------- yjit/src/core.rs | 22 +++++++--- yjit/src/cruby.rs | 1 + yjit/src/cruby_bindings.inc.rs | 3 +- yjit/src/yjit.rs | 11 +++-- 6 files changed, 108 insertions(+), 27 deletions(-) (limited to 'yjit') diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index d83e948776..29f39a7cb4 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -326,6 +326,7 @@ fn main() { .allowlist_function("rb_yjit_assert_holding_vm_lock") .allowlist_function("rb_yjit_sendish_sp_pops") .allowlist_function("rb_yjit_invokeblock_sp_pops") + .allowlist_function("rb_yjit_set_exception_return") .allowlist_type("robject_offsets") .allowlist_type("rstring_offsets") @@ -443,6 +444,7 @@ fn main() { .allowlist_function("rb_yjit_array_len") .allowlist_function("rb_obj_class") .allowlist_function("rb_obj_is_proc") + .allowlist_function("rb_vm_base_ptr") // We define VALUE manually, don't import it .blocklist_type("VALUE") diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 721b2a9b07..39fc3fb569 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -18,7 +18,7 @@ use std::cmp::min; use std::collections::HashMap; use std::ffi::CStr; use std::mem; -use std::os::raw::{c_int}; +use std::os::raw::c_int; use std::ptr; use std::rc::Rc; use std::slice; @@ -405,7 +405,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { // to the interpreter when it cannot service a stub by generating new code. // Before coming here, branch_stub_hit() takes care of fully reconstructing // interpreter state. -fn gen_code_for_exit_from_stub(ocb: &mut OutlinedCb) -> CodePtr { +fn gen_stub_exit(ocb: &mut OutlinedCb) -> CodePtr { let ocb = ocb.unwrap(); let code_ptr = ocb.get_write_ptr(); let mut asm = Assembler::new(); @@ -617,6 +617,38 @@ fn gen_leave_exit(ocb: &mut OutlinedCb) -> CodePtr { return code_ptr; } +// Increment SP and transfer the execution to the interpreter after jit_exec_exception(). +// On jit_exec_exception(), you need to return Qundef to keep executing caller non-FINISH +// frames on the interpreter. You also need to increment SP to push the return value to +// the caller's stack, which is different from gen_stub_exit(). +fn gen_leave_exception(ocb: &mut OutlinedCb) -> CodePtr { + let ocb = ocb.unwrap(); + let code_ptr = ocb.get_write_ptr(); + let mut asm = Assembler::new(); + + // Every exit to the interpreter should be counted + gen_counter_incr(&mut asm, Counter::leave_interp_return); + + asm.comment("increment SP of the caller"); + let sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); + let new_sp = asm.add(sp, SIZEOF_VALUE.into()); + asm.mov(sp, new_sp); + + asm.comment("exit from exception"); + asm.cpop_into(SP); + asm.cpop_into(EC); + asm.cpop_into(CFP); + + asm.frame_teardown(); + + // Execute vm_exec_core + asm.cret(Qundef.into()); + + asm.compile(ocb, None); + + return code_ptr; +} + // Generate a runtime guard that ensures the PC is at the expected // instruction index in the iseq, otherwise takes an entry stub // that generates another check and entry. @@ -647,7 +679,15 @@ pub fn gen_entry_chain_guard( /// Compile an interpreter entry block to be inserted into an iseq /// Returns None if compilation fails. -pub fn gen_entry_prologue(cb: &mut CodeBlock, ocb: &mut OutlinedCb, iseq: IseqPtr, insn_idx: u16) -> Option { +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See [jit_compile_exception] for details. +pub fn gen_entry_prologue( + cb: &mut CodeBlock, + ocb: &mut OutlinedCb, + iseq: IseqPtr, + insn_idx: u16, + jit_exception: bool, +) -> Option { let code_ptr = cb.get_write_ptr(); let mut asm = Assembler::new(); @@ -672,19 +712,36 @@ pub fn gen_entry_prologue(cb: &mut CodeBlock, ocb: &mut OutlinedCb, iseq: IseqPt asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); // Setup cfp->jit_return - asm.mov( - Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), - Opnd::const_ptr(CodegenGlobals::get_leave_exit_code().raw_ptr()), - ); + // If this is an exception handler entry point + if jit_exception { + // On jit_exec_exception(), it's NOT safe to return a non-Qundef value + // from a non-FINISH frame. This function fixes that problem. + // See [jit_compile_exception] for details. + asm.ccall( + rb_yjit_set_exception_return as *mut u8, + vec![ + CFP, + Opnd::const_ptr(CodegenGlobals::get_leave_exit_code().raw_ptr()), + Opnd::const_ptr(CodegenGlobals::get_leave_exception_code().raw_ptr()), + ], + ); + } else { + // On jit_exec() or JIT_EXEC(), it's safe to return a non-Qundef value + // on the entry frame. See [jit_compile] for details. + asm.mov( + Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), + Opnd::const_ptr(CodegenGlobals::get_leave_exit_code().raw_ptr()), + ); + } - // We're compiling iseqs that we *expect* to start at `insn_idx`. But in - // the case of optional parameters, the interpreter can set the pc to a - // different location depending on the optional parameters. If an iseq - // has optional parameters, we'll add a runtime check that the PC we've + // We're compiling iseqs that we *expect* to start at `insn_idx`. + // But in the case of optional parameters or when handling exceptions, + // the interpreter can set the pc to a different location. For + // such scenarios, we'll add a runtime check that the PC we've // compiled for is the same PC that the interpreter wants us to run with. // If they don't match, then we'll jump to an entry stub and generate // another PC check and entry there. - let pending_entry = if unsafe { get_iseq_flags_has_opt(iseq) } { + let pending_entry = if unsafe { get_iseq_flags_has_opt(iseq) } || jit_exception { Some(gen_entry_chain_guard(&mut asm, ocb, iseq, insn_idx)?) } else { None @@ -8283,8 +8340,11 @@ pub struct CodegenGlobals { /// Code for exiting back to the interpreter from the leave instruction leave_exit_code: CodePtr, + /// Code for exiting back to the interpreter after handling an exception + leave_exception_code: CodePtr, + // For exiting from YJIT frame from branch_stub_hit(). - // Filled by gen_code_for_exit_from_stub(). + // Filled by gen_stub_exit(). stub_exit_code: CodePtr, // For servicing branch stubs @@ -8373,8 +8433,9 @@ impl CodegenGlobals { let ocb_start_addr = ocb.unwrap().get_write_ptr(); let leave_exit_code = gen_leave_exit(&mut ocb); + let leave_exception_code = gen_leave_exception(&mut ocb); - let stub_exit_code = gen_code_for_exit_from_stub(&mut ocb); + let stub_exit_code = gen_stub_exit(&mut ocb); let branch_stub_hit_trampoline = gen_branch_stub_hit_trampoline(&mut ocb); let entry_stub_hit_trampoline = gen_entry_stub_hit_trampoline(&mut ocb); @@ -8393,7 +8454,8 @@ impl CodegenGlobals { inline_cb: cb, outlined_cb: ocb, leave_exit_code, - stub_exit_code: stub_exit_code, + leave_exception_code, + stub_exit_code, outline_full_cfunc_return_pos: cfunc_exit_code, branch_stub_hit_trampoline, entry_stub_hit_trampoline, @@ -8513,6 +8575,10 @@ impl CodegenGlobals { CodegenGlobals::get_instance().leave_exit_code } + pub fn get_leave_exception_code() -> CodePtr { + CodegenGlobals::get_instance().leave_exception_code + } + pub fn get_stub_exit_code() -> CodePtr { CodegenGlobals::get_instance().stub_exit_code } diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 32324deb97..89061ac4e5 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -2135,12 +2135,18 @@ fn gen_block_series_body( /// Generate a block version that is an entry point inserted into an iseq /// NOTE: this function assumes that the VM lock has been taken -pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> Option { +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See [jit_compile_exception] for details. +pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exception: bool) -> Option { // Compute the current instruction index based on the current PC + let cfp = unsafe { get_ec_cfp(ec) }; let insn_idx: u16 = unsafe { - let ec_pc = get_cfp_pc(get_ec_cfp(ec)); + let ec_pc = get_cfp_pc(cfp); iseq_pc_to_insn_idx(iseq, ec_pc)? }; + let stack_size: u8 = unsafe { + u8::try_from(get_cfp_sp(cfp).offset_from(get_cfp_bp(cfp))).ok()? + }; // The entry context makes no assumptions about types let blockid = BlockId { @@ -2153,10 +2159,12 @@ pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> Option { let ocb = CodegenGlobals::get_outlined_cb(); // Write the interpreter entry prologue. Might be NULL when out of memory. - let code_ptr = gen_entry_prologue(cb, ocb, iseq, insn_idx); + let code_ptr = gen_entry_prologue(cb, ocb, iseq, insn_idx, jit_exception); // Try to generate code for the entry block - let block = gen_block_series(blockid, &Context::default(), ec, cb, ocb); + let mut ctx = Context::default(); + ctx.stack_size = stack_size; + let block = gen_block_series(blockid, &ctx, ec, cb, ocb); cb.mark_all_executable(); ocb.unwrap().mark_all_executable(); @@ -2239,6 +2247,9 @@ fn entry_stub_hit_body(entry_ptr: *const c_void, ec: EcPtr) -> Option<*const u8> let cfp = unsafe { get_ec_cfp(ec) }; let iseq = unsafe { get_cfp_iseq(cfp) }; let insn_idx = iseq_pc_to_insn_idx(iseq, unsafe { get_cfp_pc(cfp) })?; + let stack_size: u8 = unsafe { + u8::try_from(get_cfp_sp(cfp).offset_from(get_cfp_bp(cfp))).ok()? + }; let cb = CodegenGlobals::get_inline_cb(); let ocb = CodegenGlobals::get_outlined_cb(); @@ -2251,7 +2262,8 @@ fn entry_stub_hit_body(entry_ptr: *const c_void, ec: EcPtr) -> Option<*const u8> // Try to find an existing compiled version of this block let blockid = BlockId { iseq, idx: insn_idx }; - let ctx = Context::default(); + let mut ctx = Context::default(); + ctx.stack_size = stack_size; let blockref = match find_block_version(blockid, &ctx) { // If an existing block is found, generate a jump to the block. Some(blockref) => { diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index a3ce30c589..254bdb1896 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -149,6 +149,7 @@ pub use rb_get_cfp_sp as get_cfp_sp; pub use rb_get_cfp_self as get_cfp_self; pub use rb_get_cfp_ep as get_cfp_ep; pub use rb_get_cfp_ep_level as get_cfp_ep_level; +pub use rb_vm_base_ptr as get_cfp_bp; pub use rb_get_cme_def_type as get_cme_def_type; pub use rb_get_cme_def_body_attr_id as get_cme_def_body_attr_id; pub use rb_get_cme_def_body_optimized_type as get_cme_def_body_optimized_type; diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 94a9335042..5f0601f988 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1253,7 +1253,6 @@ extern "C" { pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void; pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t; pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; - pub fn rb_get_iseq_body_total_calls(iseq: *const rb_iseq_t) -> ::std::os::raw::c_ulong; pub fn rb_get_iseq_body_local_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; pub fn rb_get_iseq_body_parent_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; pub fn rb_get_iseq_body_local_table_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; @@ -1297,6 +1296,7 @@ extern "C" { pub fn rb_get_cfp_self(cfp: *mut rb_control_frame_struct) -> VALUE; pub fn rb_get_cfp_ep(cfp: *mut rb_control_frame_struct) -> *mut VALUE; pub fn rb_get_cfp_ep_level(cfp: *mut rb_control_frame_struct, lv: u32) -> *const VALUE; + pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; pub fn rb_yarv_class_of(obj: VALUE) -> VALUE; pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_str_neq_internal(str1: VALUE, str2: VALUE) -> VALUE; @@ -1340,4 +1340,5 @@ extern "C" { pub fn rb_yjit_assert_holding_vm_lock(); pub fn rb_yjit_sendish_sp_pops(ci: *const rb_callinfo) -> usize; pub fn rb_yjit_invokeblock_sp_pops(ci: *const rb_callinfo) -> usize; + pub fn rb_yjit_set_exception_return(cfp: *mut rb_control_frame_t); } diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs index a453728702..7f4cbbe18d 100644 --- a/yjit/src/yjit.rs +++ b/yjit/src/yjit.rs @@ -47,11 +47,8 @@ pub fn yjit_enabled_p() -> bool { /// Test whether we are ready to compile an ISEQ or not #[no_mangle] -pub extern "C" fn rb_yjit_threshold_hit(iseq: IseqPtr) -> bool { - +pub extern "C" fn rb_yjit_threshold_hit(_iseq: IseqPtr, total_calls: u64) -> bool { let call_threshold = get_option!(call_threshold) as u64; - let total_calls = unsafe { rb_get_iseq_body_total_calls(iseq) } as u64; - return total_calls == call_threshold; } @@ -112,8 +109,10 @@ fn rb_bug_panic_hook() { /// Called from C code to begin compiling a function /// NOTE: this should be wrapped in RB_VM_LOCK_ENTER(), rb_vm_barrier() on the C side +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See [jit_compile_exception] for details. #[no_mangle] -pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> *const u8 { +pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exception: bool) -> *const u8 { // Reject ISEQs with very large temp stacks, // this will allow us to use u8/i8 values to track stack_size and sp_offset let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) }; @@ -131,7 +130,7 @@ pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> *con return std::ptr::null(); } - let maybe_code_ptr = gen_entry_point(iseq, ec); + let maybe_code_ptr = gen_entry_point(iseq, ec, jit_exception); match maybe_code_ptr { Some(ptr) => ptr.raw_ptr(), -- cgit v1.2.3