Combine call info and cache to speed up method invocation

To perform a regular method call, the VM needs two structs,
`rb_call_info` and `rb_call_cache`. At the moment, we allocate these two
structures in separate buffers. In the worst case, the CPU needs to read
4 cache lines to complete a method call. Putting the two structures
together reduces the maximum number of cache line reads to 2.

Combining the structures also saves 8 bytes per call site as the current
layout uses separate two pointers for the call info and the call cache.
This saves about 2 MiB on Discourse.

This change improves the Optcarrot benchmark at least 3%. For more
details, see attached bugs.ruby-lang.org ticket.

Complications:
 - A new instruction attribute `comptime_sp_inc` is introduced to
 calculate SP increase at compile time without using call caches. At
 compile time, a `TS_CALLDATA` operand points to a call info struct, but
 at runtime, the same operand points to a call data struct. Instruction
 that explicitly define `sp_inc` also need to define `comptime_sp_inc`.
 - MJIT code for copying call cache becomes slightly more complicated.
 - This changes the bytecode format, which might break existing tools.

[Misc #16258]

8 files changed, 58 insertions, 22 deletions

diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb
index cc16a5f898..de35e7234a 100644
--- a/tool/ruby_vm/models/attribute.rb
+++ b/tool/ruby_vm/models/attribute.rb
@@ -21,6 +21,13 @@ class RubyVM::Attribute
     @key = opts[:name]
     @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr]
     @type = opts[:type]
+    @ope_decls = @insn.opes.map do |operand|
+      decl = operand[:decl]
+      if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA'
+        decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci')
+      end
+      decl
+    end
   end
 
   def name
@@ -32,22 +39,20 @@ class RubyVM::Attribute
   end
 
   def declaration
-    opes = @insn.opes
-    if opes.empty?
+    if @ope_decls.empty?
       argv = "void"
     else
-      argv = opes.map {|o| o[:decl] }.join(', ')
+      argv = @ope_decls.join(', ')
     end
     sprintf '%s %s(%s)', @type, name, argv
   end
 
   def definition
-    opes = @insn.opes
-    if opes.empty?
+    if @ope_decls.empty?
       argv = "void"
     else
-      argv = opes.map {|o| "MAYBE_UNUSED(#{o[:decl]})" }.join(",\n    ")
-      argv = "\n    #{argv}\n" if opes.size > 1
+      argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n    ")
+      argv = "\n    #{argv}\n" if @ope_decls.size > 1
     end
     sprintf "%s\n%s(%s)", @type, name, argv
   end
diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instructions.rb
index e0fac5ff91..b0cc83a65a 100755
--- a/tool/ruby_vm/models/bare_instructions.rb
+++ b/tool/ruby_vm/models/bare_instructions.rb
@@ -33,6 +33,7 @@ class RubyVM::BareInstructions
       h[a.key] = a
     }
     @attrs_orig = @attrs.dup
+    check_attribute_consistency
     predefine_attributes
   end
 
@@ -139,8 +140,28 @@ class RubyVM::BareInstructions
     return @pops.any? {|i| i[:name] == var[:name] }
   end
 
+  def use_call_data?
+    @use_call_data ||=
+      @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' }
+  end
+
   private
 
+  def check_attribute_consistency
+    if has_attribute?('sp_inc') \
+        && use_call_data? \
+        && !has_attribute?('comptime_sp_inc')
+      # As the call cache caches information that can only be obtained at
+      # runtime, we do not need it when compiling from AST to bytecode. This
+      # attribute defines an expression that computes the stack pointer
+      # increase based on just the call info to avoid reserving space for the
+      # call cache at compile time. In the expression, all call data operands
+      # are mapped to their call info counterpart. Additionally, all mentions
+      # of `cd` in the operand name are replaced with `ci`.
+      raise "Please define attribute `comptime_sp_inc` for `#{@name}`"
+    end
+  end
+
   def generate_attribute t, k, v
     @attrs[k] ||= RubyVM::Attribute.new \
       insn: self,                       \
diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb
index 65ddfea41d..1125c4bbf6 100644
--- a/tool/ruby_vm/models/typemap.rb
+++ b/tool/ruby_vm/models/typemap.rb
@@ -12,8 +12,7 @@
 
 RubyVM::Typemap = {
   "..."            => %w[. TS_VARIABLE],
-  "CALL_CACHE"     => %w[E TS_CALLCACHE],
-  "CALL_INFO"      => %w[C TS_CALLINFO],
+  "CALL_DATA"      => %w[C TS_CALLDATA],
   "CDHASH"         => %w[H TS_CDHASH],
   "GENTRY"         => %w[G TS_GENTRY],
   "IC"             => %w[K TS_IC],
diff --git a/tool/ruby_vm/views/_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb
index 315d695cf1..b633ab4d32 100644
--- a/tool/ruby_vm/views/_insn_stack_increase.erb
+++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb
@@ -6,11 +6,11 @@
 %# conditions mentioned  in the  file COPYING  are met.   Consult the  file for
 %# details.
 %#
-PUREFUNC(MAYBE_UNUSED(static int insn_stack_increase(int depth, int insn, const VALUE *opes)));
-PUREFUNC(static rb_snum_t insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes));
+PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes)));
+PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes));
 
 rb_snum_t
-insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
+comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
 {
     static const signed char t[] = {
 % RubyVM::Instructions.each_slice 8 do |a|
@@ -34,9 +34,19 @@ insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
         UNREACHABLE;
 % RubyVM::Instructions.each do |i|
 %   next unless i.has_attribute?('sp_inc')
+%   attr_function =
+%     if i.has_attribute?('comptime_sp_inc')
+%       "attr_comptime_sp_inc_#{i.name}"
+%     else
+%       "attr_sp_inc_#{i.name}"
+%     end
     case <%= i.bin %>:
-        return attr_sp_inc_<%= i.name %>(<%=
+        return <%= attr_function %>(<%=
           i.opes.map.with_index do |v, j|
+            if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc')
+              v = v.dup
+              v[:type] = 'CALL_INFO'
+            end
             i.cast_from_VALUE v, "opes[#{j}]"
           end.join(", ")
         %>);
@@ -45,8 +55,8 @@ insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)
 }
 
 int
-insn_stack_increase(int depth, int insn, const VALUE *opes)
+comptime_insn_stack_increase(int depth, int insn, const VALUE *opes)
 {
     enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn;
-    return depth + (int)insn_stack_increase_dispatch(itype, opes);
+    return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes);
 }
diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb
index 4488876da3..b2dea03e38 100644
--- a/tool/ruby_vm/views/_mjit_compile_insn.erb
+++ b/tool/ruby_vm/views/_mjit_compile_insn.erb
@@ -34,8 +34,8 @@
 %   case ope.fetch(:type)
 %   when 'ID'
         comment_id(f, (ID)operands[<%= i %>]);
-%   when 'CALL_INFO'
-        comment_id(f, ((CALL_INFO)operands[<%= i %>])->mid);
+%   when 'CALL_DATA'
+        comment_id(f, ((CALL_DATA)operands[<%= i %>])->ci.mid);
 %   when 'VALUE'
         if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>]));
 %   end
diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb
index da7e96581b..95e7846820 100644
--- a/tool/ruby_vm/views/_mjit_compile_send.erb
+++ b/tool/ruby_vm/views/_mjit_compile_send.erb
@@ -14,10 +14,11 @@
     MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>];
 % end
 % # compiler: Use copied cc to avoid race condition
-    CALL_CACHE cc_copy = status->cc_entries + (cc - body->cc_entries);
+    CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body);
 %
     if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) {
         const rb_iseq_t *iseq;
+        const CALL_INFO ci = &cd->ci;
         unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`.
 % if insn.name == 'send'
         argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp`
@@ -58,7 +59,7 @@
                 fprintf(f, "    {\n");
                 fprintf(f, "        struct rb_calling_info calling;\n");
 % if insn.name == 'send'
-                fprintf(f, "        calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", operands[0], operands[2]);
+                fprintf(f, "        calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq);
 % else
                 fprintf(f, "        calling.block_handler = VM_BLOCK_HANDLER_NONE;\n");
 % end
diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb
index e5793a2a70..2ca5aca7cf 100644
--- a/tool/ruby_vm/views/insns_info.inc.erb
+++ b/tool/ruby_vm/views/insns_info.inc.erb
@@ -18,5 +18,5 @@
 <%= render 'leaf_helpers' %>
 <%= render 'sp_inc_helpers' %>
 <%= render 'attributes' %>
-<%= render 'insn_stack_increase' %>
+<%= render 'comptime_insn_stack_increase' %>
 <%= render 'insn_sp_pc_dependency' %>
diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb
index d9092a756d..95e71183d9 100644
--- a/tool/ruby_vm/views/mjit_compile.inc.erb
+++ b/tool/ruby_vm/views/mjit_compile.inc.erb
@@ -56,8 +56,8 @@ switch (insn) {
 <%=   render 'mjit_compile_send', locals: { insn: insn } -%>
 %   when *send_compatible_opt_insns
 %     # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated.
-%     cc_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_CACHE' }
-      if (has_valid_method_type(status->cc_entries + ((CALL_CACHE)operands[<%= cc_index %>] - body->cc_entries))) {
+%     cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' }
+      if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[<%= cd_index %>], body))) {
 <%=       render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%>
 <%=       render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%>
           break;