common.mk: install a single header file for JIT

compilation which is created by transforming a preprocessed vm.c.
This file will be used by JIT compiler's generated code which we are
going to have from succeeding commits.

Makefile.in: generate MJIT header for UNIX environments.
win32/Makefile.sub: generate MJIT header for mswin environments.
At initial merge, we're going to support only MinGW for Windows. So the
header installed by this file won't be used for short term, but we'll
add mswin support in a half year or so, for sure.

tool/transform_mjit_header.rb: New. This script was originally written as
minimize_mjit_header.rb by Vladimir N. Makarov <vmakarov@redhat.com> for
Feature 12589.
Then I refactored a little so that it can conform CodeClimate CI which is
currently set for Ruby's GitHub repository, and fixed some bugs and ported
it to work on Windows.

Also, as original minimize_mjit_header.rb takes too long time to run,
this is modified to skip minimization step because having *static*
unused definitions does not waste compilation time on -O2 since compiler
can skip to compile unused static functions. So this does no longer
"minimize" the header and is renamed.

This header installation does NOT include a header to automatically
export symbols used by MJIT. That's because original MJIT code was
failing to export symbols in the import header in macOS environment.
But I would like to have the functionality for maintainability in the
future. I'll manually export things but it would be just an intemediate
solution.

Patch by: Vladimir N. Makarov <vmakarov@redhat.com>
Part of: Feature 12589 and 14235.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@62187 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

5 files changed, 207 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
index efc7fd5864..3fa658820e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -195,3 +195,6 @@ lcov*.info
 # /win32/
 /win32/*.ico
 /win32/.time
+
+# MJIT
+/rb_mjit_header.h
diff --git a/Makefile.in b/Makefile.in
index 48dd90d32c..bca6696c9c 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -54,6 +54,7 @@ DOCTARGETS = @RDOCTARGET@ @CAPITARGET@
 EXTOUT = @EXTOUT@
 arch_hdrdir = $(EXTOUT)/include/$(arch)
 VPATH = $(arch_hdrdir)/ruby:$(hdrdir)/ruby:$(srcdir):$(srcdir)/missing
+MJIT_MIN_HEADER = $(EXTOUT)/include/$(arch)/rb_mjit_min_header-$(RUBY_PROGRAM_VERSION).h
 
 empty =
 CC_VERSION = @CC_VERSION@
@@ -407,6 +408,15 @@ probes.@OBJEXT@: $(srcdir)/probes.d $(DTRACE_REBUILD:yes=probes.stamp)
 	$(Q) $(RM) $@
 	$(Q) $(DTRACE) -G -C $(INCFLAGS) -s $(srcdir)/probes.d -o $@ $(DTRACE_REBUILD_OBJS)
 
+rb_mjit_header.h: PHONY probes.h
+	$(ECHO) building $@
+	$(Q) $(CC) $(CFLAGS) $(XCFLAGS) $(CPPFLAGS) -DMJIT_HEADER $(srcdir)/vm.c $(COUTFLAG) $@.new -E -P -dD
+	$(Q) (cmp $@.new $@ && $(ECHO0) $@ unchanged && $(RM) $@.new) || $(MV) $@.new $@
+
+$(MJIT_MIN_HEADER): rb_mjit_header.h $(srcdir)/tool/transform_mjit_header.rb
+	$(ECHO) building $@
+	$(BASERUBY) $(srcdir)/tool/transform_mjit_header.rb "$(CC)" rb_mjit_header.h $@
+
 # DTrace static library hacks described here:
 # http://mail.opensolaris.org/pipermail/dtrace-discuss/2005-August/000207.html
 ruby-glommed.$(OBJEXT):
diff --git a/common.mk b/common.mk
index 471ae838f7..28486baa03 100644
--- a/common.mk
+++ b/common.mk
@@ -59,6 +59,7 @@ ENC_TRANS_D   = $(TIMESTAMPDIR)/.enc-trans.time
 RDOCOUT       = $(EXTOUT)/rdoc
 HTMLOUT       = $(EXTOUT)/html
 CAPIOUT       = doc/capi
+MJIT_MIN_HEADER = $(EXTOUT)/include/$(arch)/rb_mjit_min_header-$(RUBY_PROGRAM_VERSION).h
 
 INITOBJS      = dmyext.$(OBJEXT) dmyenc.$(OBJEXT)
 NORMALMAINOBJ = main.$(OBJEXT)
@@ -186,7 +187,7 @@ SHOWFLAGS = showflags
 
 all: $(SHOWFLAGS) main docs
 
-main: $(SHOWFLAGS) exts $(ENCSTATIC:static=lib)encs
+main: $(SHOWFLAGS) exts $(ENCSTATIC:static=lib)encs $(MJIT_MIN_HEADER)
 	@$(NULLCMD)
 
 .PHONY: showflags
diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb
new file mode 100644
index 0000000000..308379123b
--- /dev/null
+++ b/tool/transform_mjit_header.rb
@@ -0,0 +1,183 @@
+# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com>
+# This is a script to transform functions to static inline.
+# Usage: transform_mjit_header.rb <c-compiler> <header file> <out>
+
+require 'fileutils'
+require 'tempfile'
+
+module MJITHeader
+  ATTR_VALUE_REGEXP  = /[^()]|\([^()]*\)/
+  ATTR_REGEXP        = /__attribute__\s*\(\((#{ATTR_VALUE_REGEXP})*\)\)/
+  FUNC_HEADER_REGEXP = /\A(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/
+
+  # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times.
+  RECURSIVE_MACROS = %w[
+    RASCTRYINFO
+    RASIPADDR
+  ]
+
+  IGNORED_FUNCTIONS = [
+    'rb_equal_opt', # Not used from VM and not compilable
+  ]
+
+  # Return start..stop of last decl in CODE ending STOP
+  def self.find_decl(code, stop)
+    level = 0
+    stop.downto(0) do |i|
+      if level == 0 && stop != i && decl_found?(code, i)
+        return decl_start(code, i)..stop
+      elsif code[i] == '}'
+        level += 1
+      elsif code[i] == '{'
+        level -= 1
+      end
+    end
+    0..-1
+  end
+
+  def self.decl_found?(code, i)
+    i == 0 || code[i] == ';' || code[i] == '}'
+  end
+
+  def self.decl_start(code, i)
+    if i == 0 && code[i] != ';' && code[i] != '}'
+      0
+    else
+      i + 1
+    end
+  end
+
+  # Given DECL return the name of it, nil if failed
+  def self.decl_name_of(decl)
+    ident_regex = /\w+/
+    decl = decl.gsub(/^#.+$/, '') # remove macros
+    reduced_decl = decl.gsub(/#{ATTR_REGEXP}/, '') # remove attributes
+    su1_regex = /{[^{}]*}/
+    su2_regex = /{([^{}]|su1_regex)*}/
+    su3_regex = /{([^{}]|su2_regex)*}/ # 3 nested structs/unions is probably enough
+    reduced_decl.gsub!(/#{su3_regex}/, '') # remove strutcs/unions in the header
+    id_seq_regex = /\s*(#{ident_regex}(\s+|\s*[*]+\s*))*/
+    # Process function header:
+    match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl)
+    return match[:name] if match
+    # Process non-function declaration:
+    reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization
+    match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl);
+    return match[:name] if match
+    nil
+  end
+
+  # Return true if CC with CFLAGS compiles successfully the current code.
+  # Use STAGE in the message in case of a compilation failure
+  def self.check_code!(code, cc, cflags, stage)
+    Tempfile.open(['', '.c']) do |f|
+      f.puts code
+      f.close
+      unless system("#{cc} #{cflags} #{f.path} 2>#{File::NULL}")
+        STDERR.puts "error in #{stage} header file:"
+        system("#{cc} #{cflags} #{f.path}")
+        exit 1
+      end
+    end
+  end
+
+  # Remove unpreprocessable macros
+  def self.remove_harmful_macros!(code)
+    code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '')
+  end
+
+  # -dD outputs those macros, and it produces redefinition warnings
+  def self.remove_default_macros!(code)
+    code.gsub!(/^#define __STDC_.+$/, '')
+    code.gsub!(/^#define assert\([^\)]+\) .+$/, '')
+  end
+
+  # This makes easier to process code
+  def self.separate_macro_and_code(code)
+    code.lines.partition { |l| !l.start_with?('#') }.flatten.join('')
+  end
+
+  def self.write(code, out)
+    FileUtils.mkdir_p(File.dirname(out))
+    File.write("#{out}.new", code)
+    FileUtils.mv("#{out}.new", out)
+  end
+
+  # Note that this checks runruby. This conservatively covers platform names.
+  def self.windows?
+    RUBY_PLATFORM =~ /mswin|mingw|msys/
+  end
+end
+
+if ARGV.size != 3
+  STDERR.puts 'Usage: transform_mjit_header.rb <c-compiler> <header file> <out>'
+  exit 1
+end
+
+cc      = ARGV[0]
+code    = File.read(ARGV[1]) # Current version of the header file.
+outfile = ARGV[2]
+if cc =~ /\Acl(\z| |\.exe)/
+  cflags = '-DMJIT_HEADER -Zs'
+else
+  cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors'
+end
+
+if MJITHeader.windows?
+  MJITHeader.remove_harmful_macros!(code)
+end
+MJITHeader.remove_default_macros!(code)
+
+# Check initial file correctness
+MJITHeader.check_code!(code, cc, cflags, 'initial')
+
+if MJITHeader.windows? # transformation is broken with Windows headers for now
+  STDERR.puts "\nSkipped transforming external functions to static on Windows."
+  MJITHeader.write(code, outfile)
+  exit 0
+end
+STDERR.puts "\nTransforming external functions to static:"
+
+code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW
+stop_pos     = code.match(/^#/).begin(0) # See `separate_macro_and_code`. This ignores proprocessors.
+extern_names = []
+
+# This loop changes function declarations to static inline.
+loop do
+  decl_range = MJITHeader.find_decl(code, stop_pos)
+  break if decl_range.end < 0
+
+  stop_pos = decl_range.begin - 1
+  decl = code[decl_range]
+  decl_name = MJITHeader.decl_name_of(decl)
+
+  if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)
+    STDERR.puts "transform_mjit_header: changing definition of '#{decl_name}' to declaration"
+    code[decl_range] = decl.sub(/{.+}/m, ';')
+  elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/)
+    decl.sub!(/(extern|static|inline) /, ' ')
+    unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc.
+      STDERR.puts "transform_mjit_header: making declaration of '#{decl_name}' static inline"
+    end
+
+    code[decl_range] = "static inline #{decl}"
+  elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/
+    extern_names << decl_name
+    decl[match.begin(0)...match.end(0)] = ''
+
+    if decl =~ /static/
+      STDERR.puts "warning: a static decl inside external definition of '#{decl_name}'"
+    end
+
+    header.sub!(/(extern|inline) /, ' ')
+    unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc.
+      STDERR.puts "transform_mjit_header: making external definition of '#{decl_name}' static inline"
+    end
+    code[decl_range] = "static inline #{header}#{decl}"
+  end
+end
+
+# Check the final file correctness
+MJITHeader.check_code!(code, cc, cflags, 'final')
+
+MJITHeader.write(code, outfile)
diff --git a/win32/Makefile.sub b/win32/Makefile.sub
index cf900fbe35..cbd36e0405 100644
--- a/win32/Makefile.sub
+++ b/win32/Makefile.sub
@@ -1186,6 +1186,15 @@ probes.h: {$(VPATH)}probes.dmyh
 #include "$(*F).dmyh"
 <<KEEP
 
+rb_mjit_header.h: PHONY probes.h
+	$(ECHO) building $@
+	$(Q) $(CC) $(CFLAGS) $(XCFLAGS) $(CPPFLAGS) -DMJIT_HEADER $(srcdir)/vm.c -P
+	$(Q) (cmp vm.i $@ && $(ECHO0) $@ unchanged && $(RM) vm.i) || $(MV) vm.i $@
+
+$(MJIT_MIN_HEADER): rb_mjit_header.h $(srcdir)/tool/transform_mjit_header.rb
+	$(ECHO) building $@
+	$(BASERUBY) $(srcdir)/tool/transform_mjit_header.rb "$(CC)" rb_mjit_header.h $@
+
 INSNS	= opt_sc.inc optinsn.inc optunifs.inc insns.inc insns_info.inc \
 	  vmtc.inc vm.inc