YJIT: Fix large ISeq rejection (#7576)

We crashed in some edge cases due to the recent change to not compile
encoded iseqs that are larger than `u16::MAX`.

- Match the C signature of rb_yjit_constant_ic_update() and clamp down
  to `IseqIdx` size
- Return failure instead of panicking with `unwrap()` in codegen when
  the iseq is too large

Co-authored-by: Maxime Chevalier-Boisvert <maxime.chevalierboisvert@shopify.com>
Co-authored-by: Noah Gibbs <noah.gibbs@shopify.com>

1 files changed, 10 insertions, 1 deletions

diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs
index 5156ca6a26..c93213b484 100644
--- a/yjit/src/invariants.rs
+++ b/yjit/src/invariants.rs
@@ -389,12 +389,21 @@ pub fn block_assumptions_free(blockref: BlockRef) {
 /// Invalidate the block for the matching opt_getinlinecache so it could regenerate code
 /// using the new value in the constant cache.
 #[no_mangle]
-pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: u16) {
+pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: std::os::raw::c_uint) {
     // If YJIT isn't enabled, do nothing
     if !yjit_enabled_p() {
         return;
     }
 
+    // Try to downcast the iseq index
+    let insn_idx: IseqIdx = if let Ok(idx) = insn_idx.try_into() {
+        idx
+    } else {
+        // The index is too large, YJIT can't possibily have code for it,
+        // so there is nothing to invalidate.
+        return;
+    };
+
     if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_yjit_multi_ractor_p() } {
         // We can't generate code in these situations, so no need to invalidate.
         // See gen_opt_getinlinecache.