Optimize exivar access on JIT-ed getivar

JIT support of dd723771c11.

$ benchmark-driver -v --rbenv 'before;before --jit;after --jit' benchmark/mjit_exivar.yml --repeat-count=4
before: ruby 2.8.0dev (2020-03-30T12:32:26Z master e5db3da9d3) [x86_64-linux]
before --jit: ruby 2.8.0dev (2020-03-30T12:32:26Z master e5db3da9d3) +JIT [x86_64-linux]
after --jit: ruby 2.8.0dev (2020-03-31T05:57:24Z mjit-exivar 128625baec) +JIT [x86_64-linux]
Calculating -------------------------------------
                         before  before --jit  after --jit
         mjit_exivar    57.944M       53.579M      54.471M i/s -    200.000M times in 3.451588s 3.732772s 3.671687s

Comparison:
                      mjit_exivar
              before:  57944345.1 i/s
         after --jit:  54470876.7 i/s - 1.06x  slower
        before --jit:  53579483.4 i/s - 1.08x  slower

1 files changed, 30 insertions, 3 deletions

diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb
index 57f8d14b76..85850b4446 100644
--- a/tool/ruby_vm/views/_mjit_compile_ivar.erb
+++ b/tool/ruby_vm/views/_mjit_compile_ivar.erb
@@ -16,17 +16,16 @@
 % # compiler: Use copied IVC to avoid race condition
     IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache;
 %
-% # compiler: Consider cfp->self as T_OBJECT if ic_copy->ic_serial is set
     if (!status->compile_info->disable_ivar_cache && ic_copy->ic_serial) {
 % # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`.
 % # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%>
 %
-% # JIT: prepare vm_getivar's arguments and variables
+% # JIT: prepare vm_getivar/vm_setivar arguments and variables
         fprintf(f, "{\n");
         fprintf(f, "    VALUE obj = GET_SELF();\n");
         fprintf(f, "    const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->ic_serial);
         fprintf(f, "    const st_index_t index = %"PRIuSIZE";\n", ic_copy->index);
-% # JIT: cache hit path of vm_getivar, or cancel JIT.
+% # JIT: cache hit path of vm_getivar/vm_setivar, or cancel JIT (recompile it with exivar)
 % if insn.name == 'setinstancevariable'
         fprintf(f, "    VALUE val = stack[%d];\n", b->stack_size - 1);
         fprintf(f, "    if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN(obj))) {\n");
@@ -50,5 +49,33 @@
         fprintf(f, "}\n");
         break;
     }
+% if insn.name == 'getinstancevariable'
+    else if (!status->compile_info->disable_exivar_cache && ic_copy->ic_serial) {
+% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`.
+% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%>
+%
+% # JIT: prepare vm_getivar's arguments and variables
+        fprintf(f, "{\n");
+        fprintf(f, "    VALUE obj = GET_SELF();\n");
+        fprintf(f, "    const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->ic_serial);
+        fprintf(f, "    const st_index_t index = %"PRIuSIZE";\n", ic_copy->index);
+% # JIT: cache hit path of vm_getivar, or cancel JIT (recompile it without any ivar optimization)
+        fprintf(f, "    struct gen_ivtbl *ivtbl;\n");
+        fprintf(f, "    VALUE val;\n");
+        fprintf(f, "    if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && st_lookup(rb_ivar_generic_ivtbl(), (st_data_t)obj, (st_data_t *)&ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n");
+        fprintf(f, "        stack[%d] = val;\n", b->stack_size);
+        fprintf(f, "    }\n");
+        fprintf(f, "    else {\n");
+        fprintf(f, "        reg_cfp->pc = original_body_iseq + %d;\n", pos);
+        fprintf(f, "        reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size);
+        fprintf(f, "        goto exivar_cancel;\n");
+        fprintf(f, "    }\n");
+
+% # compiler: Move JIT compiler's internal stack pointer
+        b->stack_size += <%= insn.call_attribute('sp_inc') %>;
+        fprintf(f, "}\n");
+        break;
+    }
+% end
 }
 #endif // OPT_IC_FOR_IVAR