YJIT: Fix large ISeq rejection (#7576)

We crashed in some edge cases due to the recent change to not compile
encoded iseqs that are larger than `u16::MAX`.

- Match the C signature of rb_yjit_constant_ic_update() and clamp down
  to `IseqIdx` size
- Return failure instead of panicking with `unwrap()` in codegen when
  the iseq is too large

Co-authored-by: Maxime Chevalier-Boisvert <maxime.chevalierboisvert@shopify.com>
Co-authored-by: Noah Gibbs <noah.gibbs@shopify.com>

bootstraptest/test_yjit.rb

@@ -3677,3 +3677,16 @@ assert_equal "foo", %q{
 
   literal("foo")
 }
+
+# regression test for accidentally having a parameter truncated
+# due to Rust/C signature mismatch. Used to crash with
+# > [BUG] rb_vm_insn_addr2insn: invalid insn address ...
+# or
+# > ... `Err` value: TryFromIntError(())'
+assert_normal_exit %q{
+  n = 16384
+  eval(
+    "def foo(arg); " + "_=arg;" * n + '_=1;' + "Object; end"
+  )
+  foo 1
+}

yjit/src/codegen.rs

@@ -781,8 +781,13 @@ pub fn gen_single_block(
     // Instruction sequence to compile
     let iseq = blockid.iseq;
     let iseq_size = unsafe { get_iseq_encoded_size(iseq) };
-    let iseq_size: u16 = iseq_size.try_into().unwrap();
-    let mut insn_idx: u16 = blockid.idx;
+    let iseq_size: IseqIdx = if let Ok(size) = iseq_size.try_into() {
+        size
+    } else {
+        // ISeq too large to compile
+        return Err(());
+    };
+    let mut insn_idx: IseqIdx = blockid.idx;
 
     // Initialize a JIT state object
     let mut jit = JITState::new(blockid, ctx.clone(), cb.get_write_ptr(), ec);

yjit/src/invariants.rs

@@ -389,12 +389,21 @@ pub fn block_assumptions_free(blockref: BlockRef) {
 /// Invalidate the block for the matching opt_getinlinecache so it could regenerate code
 /// using the new value in the constant cache.
 #[no_mangle]
-pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: u16) {
+pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: std::os::raw::c_uint) {
     // If YJIT isn't enabled, do nothing
     if !yjit_enabled_p() {
         return;
     }
 
+    // Try to downcast the iseq index
+    let insn_idx: IseqIdx = if let Ok(idx) = insn_idx.try_into() {
+        idx
+    } else {
+        // The index is too large, YJIT can't possibily have code for it,
+        // so there is nothing to invalidate.
+        return;
+    };
+
     if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_yjit_multi_ractor_p() } {
         // We can't generate code in these situations, so no need to invalidate.
         // See gen_opt_getinlinecache.