YJIT: Plug native stack overflow

Previously, TestStack#test_machine_stack_size failed pretty consistently
on ARM64 macOS, with Rust code and part of the interpreter used for
per-instruction fallback (rb_vm_invokeblock() and friends) touching the
stack guard page and crashing with SEGV. I've also seen the same test
fail on x64 Linux, though with a different symptom.

1 files changed, 5 insertions, 0 deletions

diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs
index 97799923d2..2aed3c6a4b 100644
--- a/yjit/src/yjit.rs
+++ b/yjit/src/yjit.rs
@@ -114,6 +114,11 @@ fn rb_bug_panic_hook() {
 /// See [jit_compile_exception] for details.
 #[no_mangle]
 pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exception: bool) -> *const u8 {
+    // Don't compile when there is insufficient native stack space
+    if unsafe { rb_ec_stack_check(ec as _) } != 0 {
+        return std::ptr::null();
+    }
+
     // Reject ISEQs with very large temp stacks,
     // this will allow us to use u8/i8 values to track stack_size and sp_offset
     let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) };