From 89e7997622038f82115f34dbb4ea382e02bed163 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Tue, 30 Jul 2019 21:36:05 -0400 Subject: Combine call info and cache to speed up method invocation To perform a regular method call, the VM needs two structs, `rb_call_info` and `rb_call_cache`. At the moment, we allocate these two structures in separate buffers. In the worst case, the CPU needs to read 4 cache lines to complete a method call. Putting the two structures together reduces the maximum number of cache line reads to 2. Combining the structures also saves 8 bytes per call site as the current layout uses separate two pointers for the call info and the call cache. This saves about 2 MiB on Discourse. This change improves the Optcarrot benchmark at least 3%. For more details, see attached bugs.ruby-lang.org ticket. Complications: - A new instruction attribute `comptime_sp_inc` is introduced to calculate SP increase at compile time without using call caches. At compile time, a `TS_CALLDATA` operand points to a call info struct, but at runtime, the same operand points to a call data struct. Instruction that explicitly define `sp_inc` also need to define `comptime_sp_inc`. - MJIT code for copying call cache becomes slightly more complicated. - This changes the bytecode format, which might break existing tools. [Misc #16258] --- tool/ruby_vm/models/attribute.rb | 19 ++++--- tool/ruby_vm/models/bare_instructions.rb | 21 ++++++++ tool/ruby_vm/models/typemap.rb | 3 +- .../views/_comptime_insn_stack_increase.erb | 62 ++++++++++++++++++++++ tool/ruby_vm/views/_insn_stack_increase.erb | 52 ------------------ tool/ruby_vm/views/_mjit_compile_insn.erb | 4 +- tool/ruby_vm/views/_mjit_compile_send.erb | 5 +- tool/ruby_vm/views/insns_info.inc.erb | 2 +- tool/ruby_vm/views/mjit_compile.inc.erb | 4 +- 9 files changed, 104 insertions(+), 68 deletions(-) create mode 100644 tool/ruby_vm/views/_comptime_insn_stack_increase.erb delete mode 100644 tool/ruby_vm/views/_insn_stack_increase.erb (limited to 'tool/ruby_vm') diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index cc16a5f898..de35e7234a 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -21,6 +21,13 @@ class RubyVM::Attribute @key = opts[:name] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @type = opts[:type] + @ope_decls = @insn.opes.map do |operand| + decl = operand[:decl] + if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' + decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') + end + decl + end end def name @@ -32,22 +39,20 @@ class RubyVM::Attribute end def declaration - opes = @insn.opes - if opes.empty? + if @ope_decls.empty? argv = "void" else - argv = opes.map {|o| o[:decl] }.join(', ') + argv = @ope_decls.join(', ') end sprintf '%s %s(%s)', @type, name, argv end def definition - opes = @insn.opes - if opes.empty? + if @ope_decls.empty? argv = "void" else - argv = opes.map {|o| "MAYBE_UNUSED(#{o[:decl]})" }.join(",\n ") - argv = "\n #{argv}\n" if opes.size > 1 + argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n ") + argv = "\n #{argv}\n" if @ope_decls.size > 1 end sprintf "%s\n%s(%s)", @type, name, argv end diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instructions.rb index e0fac5ff91..b0cc83a65a 100755 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instructions.rb @@ -33,6 +33,7 @@ class RubyVM::BareInstructions h[a.key] = a } @attrs_orig = @attrs.dup + check_attribute_consistency predefine_attributes end @@ -139,8 +140,28 @@ class RubyVM::BareInstructions return @pops.any? {|i| i[:name] == var[:name] } end + def use_call_data? + @use_call_data ||= + @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } + end + private + def check_attribute_consistency + if has_attribute?('sp_inc') \ + && use_call_data? \ + && !has_attribute?('comptime_sp_inc') + # As the call cache caches information that can only be obtained at + # runtime, we do not need it when compiling from AST to bytecode. This + # attribute defines an expression that computes the stack pointer + # increase based on just the call info to avoid reserving space for the + # call cache at compile time. In the expression, all call data operands + # are mapped to their call info counterpart. Additionally, all mentions + # of `cd` in the operand name are replaced with `ci`. + raise "Please define attribute `comptime_sp_inc` for `#{@name}`" + end + end + def generate_attribute t, k, v @attrs[k] ||= RubyVM::Attribute.new \ insn: self, \ diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index 65ddfea41d..1125c4bbf6 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -12,8 +12,7 @@ RubyVM::Typemap = { "..." => %w[. TS_VARIABLE], - "CALL_CACHE" => %w[E TS_CALLCACHE], - "CALL_INFO" => %w[C TS_CALLINFO], + "CALL_DATA" => %w[C TS_CALLDATA], "CDHASH" => %w[H TS_CDHASH], "GENTRY" => %w[G TS_GENTRY], "IC" => %w[K TS_IC], diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb new file mode 100644 index 0000000000..b633ab4d32 --- /dev/null +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -0,0 +1,62 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%# +PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); +PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); + +rb_snum_t +comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) +{ + static const signed char t[] = { +% RubyVM::Instructions.each_slice 8 do |a| + <%= a.map { |i| + if i.has_attribute?('sp_inc') + '-127' + else + sprintf("%4d", i.rets.size - i.pops.size) + end + }.join(', ') -%>, +% end + }; + signed char c = t[insn]; + + ASSERT_VM_INSTRUCTION_SIZE(t); + if (c != -127) { + return c; + } + else switch(insn) { + default: + UNREACHABLE; +% RubyVM::Instructions.each do |i| +% next unless i.has_attribute?('sp_inc') +% attr_function = +% if i.has_attribute?('comptime_sp_inc') +% "attr_comptime_sp_inc_#{i.name}" +% else +% "attr_sp_inc_#{i.name}" +% end + case <%= i.bin %>: + return <%= attr_function %>(<%= + i.opes.map.with_index do |v, j| + if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') + v = v.dup + v[:type] = 'CALL_INFO' + end + i.cast_from_VALUE v, "opes[#{j}]" + end.join(", ") + %>); +% end + } +} + +int +comptime_insn_stack_increase(int depth, int insn, const VALUE *opes) +{ + enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; + return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes); +} diff --git a/tool/ruby_vm/views/_insn_stack_increase.erb b/tool/ruby_vm/views/_insn_stack_increase.erb deleted file mode 100644 index 315d695cf1..0000000000 --- a/tool/ruby_vm/views/_insn_stack_increase.erb +++ /dev/null @@ -1,52 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static int insn_stack_increase(int depth, int insn, const VALUE *opes))); -PUREFUNC(static rb_snum_t insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); - -rb_snum_t -insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) -{ - static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, -% end - }; - signed char c = t[insn]; - - ASSERT_VM_INSTRUCTION_SIZE(t); - if (c != -127) { - return c; - } - else switch(insn) { - default: - UNREACHABLE; -% RubyVM::Instructions.each do |i| -% next unless i.has_attribute?('sp_inc') - case <%= i.bin %>: - return attr_sp_inc_<%= i.name %>(<%= - i.opes.map.with_index do |v, j| - i.cast_from_VALUE v, "opes[#{j}]" - end.join(", ") - %>); -% end - } -} - -int -insn_stack_increase(int depth, int insn, const VALUE *opes) -{ - enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; - return depth + (int)insn_stack_increase_dispatch(itype, opes); -} diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb index 4488876da3..b2dea03e38 100644 --- a/tool/ruby_vm/views/_mjit_compile_insn.erb +++ b/tool/ruby_vm/views/_mjit_compile_insn.erb @@ -34,8 +34,8 @@ % case ope.fetch(:type) % when 'ID' comment_id(f, (ID)operands[<%= i %>]); -% when 'CALL_INFO' - comment_id(f, ((CALL_INFO)operands[<%= i %>])->mid); +% when 'CALL_DATA' + comment_id(f, ((CALL_DATA)operands[<%= i %>])->ci.mid); % when 'VALUE' if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); % end diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb index da7e96581b..95e7846820 100644 --- a/tool/ruby_vm/views/_mjit_compile_send.erb +++ b/tool/ruby_vm/views/_mjit_compile_send.erb @@ -14,10 +14,11 @@ MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; % end % # compiler: Use copied cc to avoid race condition - CALL_CACHE cc_copy = status->cc_entries + (cc - body->cc_entries); + CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); % if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; + const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. % if insn.name == 'send' argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp` @@ -58,7 +59,7 @@ fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); % if insn.name == 'send' - fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", operands[0], operands[2]); + fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); % else fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); % end diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index e5793a2a70..2ca5aca7cf 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -18,5 +18,5 @@ <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> <%= render 'attributes' %> -<%= render 'insn_stack_increase' %> +<%= render 'comptime_insn_stack_increase' %> <%= render 'insn_sp_pc_dependency' %> diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb index d9092a756d..95e71183d9 100644 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ b/tool/ruby_vm/views/mjit_compile.inc.erb @@ -56,8 +56,8 @@ switch (insn) { <%= render 'mjit_compile_send', locals: { insn: insn } -%> % when *send_compatible_opt_insns % # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. -% cc_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_CACHE' } - if (has_valid_method_type(status->cc_entries + ((CALL_CACHE)operands[<%= cc_index %>] - body->cc_entries))) { +% cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' } + if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[<%= cd_index %>], body))) { <%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> <%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> break; -- cgit v1.2.3