Introduce an "Inline IVAR cache" struct

This commit introduces an "inline ivar cache" struct.  The reason we
need this is so compaction can differentiate from an ivar cache and a
regular inline cache.  Regular inline caches contain references to
`VALUE` and ivar caches just contain references to the ivar index.  With
this new struct we can easily update references for inline caches (but
not inline var caches as they just contain an int)

7 files changed, 33 insertions, 21 deletions

diff --git a/compile.c b/compile.c
index e261a38d7d..aee96724a1 100644
--- a/compile.c
+++ b/compile.c
@@ -2190,6 +2190,7 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
 			FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
                         /* fall through */
 		      case TS_IC: /* inline cache */
+		      case TS_IVC: /* inline ivar cache */
 			{
 			    unsigned int ic_index = FIX2UINT(operands[j]);
 			    IC ic = (IC)&body->is_entries[ic_index];
@@ -8648,6 +8649,7 @@ insn_data_to_s_detail(INSN *iobj)
 		    break;
 		}
 	      case TS_IC:	/* inline cache */
+	      case TS_IVC:	/* inline ivar cache */
 	      case TS_ISE:	/* inline storage entry */
 		rb_str_catf(str, "<ic:%d>", FIX2INT(OPERAND_AT(iobj, j)));
 		break;
@@ -9035,6 +9037,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor,
 			FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
                         /* fall through */
 		      case TS_IC:
+                      case TS_IVC:  /* inline ivar cache */
 			argv[j] = op;
 			if (NUM2UINT(op) >= iseq->body->is_size) {
 			    iseq->body->is_size = NUM2INT(op) + 1;
@@ -9883,6 +9886,7 @@ ibf_dump_code(struct ibf_dump *dump, const rb_iseq_t *iseq)
                 wv = (VALUE)ibf_dump_iseq(dump, (const rb_iseq_t *)op);
                 break;
               case TS_IC:
+              case TS_IVC:
               case TS_ISE:
                 {
                     unsigned int i;
@@ -9974,6 +9978,7 @@ ibf_load_code(const struct ibf_load *load, const rb_iseq_t *iseq, ibf_offset_t b
                 FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
                 /* fall through */
               case TS_IC:
+              case TS_IVC:
                 {
                     VALUE op = ibf_load_small_value(load, &reading_pos);
                     code[code_index] = (VALUE)&is_entries[op];
diff --git a/insns.def b/insns.def
index 9bad6768f0..bd1bffbe02 100644
--- a/insns.def
+++ b/insns.def
@@ -207,7 +207,7 @@ setspecial
 /* Get value of instance variable id of self. */
 DEFINE_INSN
 getinstancevariable
-(ID id, IC ic)
+(ID id, IVC ic)
 ()
 (VALUE val)
 /* "instance variable not initialized" warning can be hooked. */
@@ -219,7 +219,7 @@ getinstancevariable
 /* Set value of instance variable id of self to val. */
 DEFINE_INSN
 setinstancevariable
-(ID id, IC ic)
+(ID id, IVC ic)
 (VALUE val)
 ()
 // attr bool leaf = false; /* has rb_check_frozen_internal() */
@@ -1040,7 +1040,7 @@ opt_getinlinecache
 (VALUE val)
 {
     if (vm_ic_hit_p(ic, GET_EP())) {
-	val = ic->ic_value.value;
+	val = ic->value;
 	JUMP(dst);
     }
     else {
diff --git a/iseq.c b/iseq.c
index 427739859b..7fbb906525 100644
--- a/iseq.c
+++ b/iseq.c
@@ -1917,6 +1917,7 @@ rb_insn_operand_intern(const rb_iseq_t *iseq,
 	break;
 
       case TS_IC:
+      case TS_IVC:
       case TS_ISE:
 	ret = rb_sprintf("<is:%"PRIdPTRDIFF">", (union iseq_inline_storage_entry *)op - iseq->body->is_entries);
 	break;
@@ -2741,6 +2742,7 @@ iseq_data_to_ary(const rb_iseq_t *iseq)
 		}
 		break;
 	      case TS_IC:
+              case TS_IVC:
 	      case TS_ISE:
 		{
 		    union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)*seq;
diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb
index 015aa05632..c4b13f67f9 100644
--- a/tool/ruby_vm/models/typemap.rb
+++ b/tool/ruby_vm/models/typemap.rb
@@ -16,6 +16,7 @@ RubyVM::Typemap = {
   "CDHASH"         => %w[H TS_CDHASH],
   "GENTRY"         => %w[G TS_GENTRY],
   "IC"             => %w[K TS_IC],
+  "IVC"            => %w[A TS_IVC],
   "ID"             => %w[I TS_ID],
   "ISE"            => %w[T TS_ISE],
   "ISEQ"           => %w[S TS_ISEQ],
diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb
index 4b4dcec828..57f8d14b76 100644
--- a/tool/ruby_vm/views/_mjit_compile_ivar.erb
+++ b/tool/ruby_vm/views/_mjit_compile_ivar.erb
@@ -13,8 +13,8 @@
 % insn.opes.each_with_index do |ope, i|
     MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>];
 % end
-% # compiler: Use copied IC to avoid race condition
-    IC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->cache;
+% # compiler: Use copied IVC to avoid race condition
+    IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache;
 %
 % # compiler: Consider cfp->self as T_OBJECT if ic_copy->ic_serial is set
     if (!status->compile_info->disable_ivar_cache && ic_copy->ic_serial) {
@@ -25,7 +25,7 @@
         fprintf(f, "{\n");
         fprintf(f, "    VALUE obj = GET_SELF();\n");
         fprintf(f, "    const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->ic_serial);
-        fprintf(f, "    const st_index_t index = %"PRIuSIZE";\n", ic_copy->ic_value.index);
+        fprintf(f, "    const st_index_t index = %"PRIuSIZE";\n", ic_copy->index);
 % # JIT: cache hit path of vm_getivar, or cancel JIT.
 % if insn.name == 'setinstancevariable'
         fprintf(f, "    VALUE val = stack[%d];\n", b->stack_size - 1);
diff --git a/vm_core.h b/vm_core.h
index b0787aaa7a..f7ec156cc5 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -220,10 +220,12 @@ typedef struct rb_compile_option_struct rb_compile_option_t;
 struct iseq_inline_cache_entry {
     rb_serial_t ic_serial;
     const rb_cref_t *ic_cref;
-    union {
-	size_t index;
-	VALUE value;
-    } ic_value;
+    VALUE value;
+};
+
+struct iseq_inline_iv_cache_entry {
+    rb_serial_t ic_serial;
+    size_t index;
 };
 
 union iseq_inline_storage_entry {
@@ -232,6 +234,7 @@ union iseq_inline_storage_entry {
 	VALUE value;
     } once;
     struct iseq_inline_cache_entry cache;
+    struct iseq_inline_iv_cache_entry iv_cache;
 };
 
 struct rb_call_info_kw_arg {
@@ -1122,6 +1125,7 @@ enum vm_svar_index {
 
 /* inline cache */
 typedef struct iseq_inline_cache_entry *IC;
+typedef struct iseq_inline_iv_cache_entry *IVC;
 typedef union iseq_inline_storage_entry *ISE;
 typedef struct rb_call_info *CALL_INFO;
 typedef struct rb_call_cache *CALL_CACHE;
diff --git a/vm_insnhelper.c b/vm_insnhelper.c
index 1c54de6375..c8ea3f9b1b 100644
--- a/vm_insnhelper.c
+++ b/vm_insnhelper.c
@@ -1008,9 +1008,9 @@ vm_search_const_defined_class(const VALUE cbase, ID id)
     return 0;
 }
 
-ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, IC, struct rb_call_cache *, int));
+ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, IVC, struct rb_call_cache *, int));
 static inline VALUE
-vm_getivar(VALUE obj, ID id, IC ic, struct rb_call_cache *cc, int is_attr)
+vm_getivar(VALUE obj, ID id, IVC ic, struct rb_call_cache *cc, int is_attr)
 {
 #if OPT_IC_FOR_IVAR
     VALUE val = Qundef;
@@ -1022,7 +1022,7 @@ vm_getivar(VALUE obj, ID id, IC ic, struct rb_call_cache *cc, int is_attr)
                     RB_DEBUG_COUNTER_INC_UNLESS(ivar_get_ic_miss_unset, cc->aux.index > 0) :
                     RB_DEBUG_COUNTER_INC_UNLESS(ivar_get_ic_miss_serial,
                                                 ic->ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass)))) {
-        st_index_t index = !is_attr ? ic->ic_value.index : (cc->aux.index - 1);
+        st_index_t index = !is_attr ? ic->index : (cc->aux.index - 1);
 
         RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
 
@@ -1056,7 +1056,7 @@ vm_getivar(VALUE obj, ID id, IC ic, struct rb_call_cache *cc, int is_attr)
 	    if (iv_index_tbl) {
 		if (st_lookup(iv_index_tbl, id, &index)) {
                     if (!is_attr) {
-                        ic->ic_value.index = index;
+                        ic->index = index;
                         ic->ic_serial = RCLASS_SERIAL(RBASIC(obj)->klass);
                     }
                     else { /* call_info */
@@ -1108,7 +1108,7 @@ vm_getivar(VALUE obj, ID id, IC ic, struct rb_call_cache *cc, int is_attr)
 }
 
 static inline VALUE
-vm_setivar(VALUE obj, ID id, VALUE val, IC ic, struct rb_call_cache *cc, int is_attr)
+vm_setivar(VALUE obj, ID id, VALUE val, IVC ic, struct rb_call_cache *cc, int is_attr)
 {
 #if OPT_IC_FOR_IVAR
     rb_check_frozen_internal(obj);
@@ -1121,7 +1121,7 @@ vm_setivar(VALUE obj, ID id, VALUE val, IC ic, struct rb_call_cache *cc, int is_
 	    (!is_attr && RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_serial, ic->ic_serial == RCLASS_SERIAL(klass))) ||
 	    ( is_attr && RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_unset, cc->aux.index > 0)))) {
 	    VALUE *ptr = ROBJECT_IVPTR(obj);
-	    index = !is_attr ? ic->ic_value.index : cc->aux.index-1;
+	    index = !is_attr ? ic->index : cc->aux.index-1;
 
 	    if (RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_oorange, index < ROBJECT_NUMIV(obj))) {
 		RB_OBJ_WRITE(obj, &ptr[index], val);
@@ -1134,7 +1134,7 @@ vm_setivar(VALUE obj, ID id, VALUE val, IC ic, struct rb_call_cache *cc, int is_
 
 	    if (iv_index_tbl && st_lookup(iv_index_tbl, (st_data_t)id, &index)) {
                 if (!is_attr) {
-                    ic->ic_value.index = index;
+                    ic->index = index;
                     ic->ic_serial = RCLASS_SERIAL(klass);
                 }
 		else if (index >= INT_MAX) {
@@ -1156,13 +1156,13 @@ vm_setivar(VALUE obj, ID id, VALUE val, IC ic, struct rb_call_cache *cc, int is_
 }
 
 static inline VALUE
-vm_getinstancevariable(VALUE obj, ID id, IC ic)
+vm_getinstancevariable(VALUE obj, ID id, IVC ic)
 {
     return vm_getivar(obj, id, ic, NULL, FALSE);
 }
 
 static inline void
-vm_setinstancevariable(VALUE obj, ID id, VALUE val, IC ic)
+vm_setinstancevariable(VALUE obj, ID id, VALUE val, IVC ic)
 {
     vm_setivar(obj, id, val, ic, 0, 0);
 }
@@ -4134,8 +4134,8 @@ vm_ic_hit_p(IC ic, const VALUE *reg_ep)
 static void
 vm_ic_update(IC ic, VALUE val, const VALUE *reg_ep)
 {
-    VM_ASSERT(ic->ic_value.value != Qundef);
-    ic->ic_value.value = val;
+    VM_ASSERT(ic->value != Qundef);
+    ic->value = val;
     ic->ic_serial = GET_GLOBAL_CONSTANT_STATE() - ruby_vm_const_missing_count;
     ic->ic_cref = vm_get_const_key_cref(reg_ep);
     ruby_vm_const_missing_count = 0;