1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
|
# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com>
# This is a script to transform functions to static inline.
# Usage: transform_mjit_header.rb <c-compiler> <header file> <out>
require 'fileutils'
require 'tempfile'
PROGRAM = File.basename($0, ".*")
module MJITHeader
ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/
ATTR_REGEXP = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/
FUNC_HEADER_REGEXP = /\A(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/
TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr)_/
# Predefined macros for compilers which are already supported by MJIT.
# We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro.
SUPPORTED_CC_MACROS = [
'__GNUC__', # gcc
'__clang__', # clang
]
# For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times.
RECURSIVE_MACROS = %w[
RASCTRYINFO
RASIPADDR
]
IGNORED_FUNCTIONS = [
'vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function.
'rb_equal_opt', # Not used from VM and not compilable
]
# Return start..stop of last decl in CODE ending STOP
def self.find_decl(code, stop)
level = 0
i = stop
while i = code.rindex(/[;{}]/, i)
if level == 0 && stop != i && decl_found?($&, i)
return decl_start($&, i)..stop
end
case $&
when '}'
level += 1
when '{'
level -= 1
end
i -= 1
end
nil
end
def self.decl_found?(code, i)
i == 0 || code == ';' || code == '}'
end
def self.decl_start(code, i)
if i == 0 && code != ';' && code != '}'
0
else
i + 1
end
end
# Given DECL return the name of it, nil if failed
def self.decl_name_of(decl)
ident_regex = /\w+/
decl = decl.gsub(/^#.+$/, '') # remove macros
reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes
su1_regex = /{[^{}]*}/
su2_regex = /{([^{}]|#{su1_regex})*}/
su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough
reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header
id_seq_regex = /\s*(#{ident_regex}(\s+|\s*[*]+\s*))*/
# Process function header:
match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl)
return match[:name] if match
# Process non-function declaration:
reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization
match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl);
return match[:name] if match
nil
end
# Return true if CC with CFLAGS compiles successfully the current code.
# Use STAGE in the message in case of a compilation failure
def self.check_code!(code, cc, cflags, stage)
with_code(code) do |path|
cmd = "#{cc} #{cflags} #{path}"
unless system(cmd, err: File::NULL)
out = IO.popen(cmd, err: [:child, :out], &:read)
STDERR.puts "error in #{stage} header file:\n#{out}"
if match = out.match(/error: conflicting types for '[^']+'/)
STDERR.puts "\nDumping information for debugging:\n"\
"[ORIGINAL_HEADER_BEGIN]-----------------\n#{File.binread(ARGV[1])}\n"\
"[ORIGINAL_HEADER_END]-----------------\n\n"\
"[TRANSFORMED_HEADER_BEGIN]-----------------\n#{code}\n"\
"[TRANSFORMED_HEADER_END]-----------------\n"
end
exit false
end
end
end
# Remove unpreprocessable macros
def self.remove_harmful_macros!(code)
code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '')
end
# -dD outputs those macros, and it produces redefinition warnings or errors
def self.remove_predefined_macros!(code)
code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2')
end
# This makes easier to process code
def self.separate_macro_and_code(code)
code.lines.partition { |l| l.start_with?('#') }.map! {|lines| lines.join('')}
end
def self.write(code, out)
FileUtils.mkdir_p(File.dirname(out))
File.binwrite("#{out}.new", code)
FileUtils.mv("#{out}.new", out)
end
# Note that this checks runruby. This conservatively covers platform names.
def self.windows?
RUBY_PLATFORM =~ /mswin|mingw|msys/
end
def self.cl_exe?(cc)
cc =~ /\Acl(\z| |\.exe)/
end
# If code has macro which only supported compilers predefine, return true.
def self.supported_header?(code)
SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ }
end
# This checks if syntax check outputs "error: conflicting types for 'restrict'".
# If it's true, this script regards platform as AIX and add -std=c99 as workaround.
def self.conflicting_types?(code, cc, cflags)
with_code(code) do |path|
cmd = "#{cc} #{cflags} #{path}"
out = IO.popen(cmd, err: [:child, :out], &:read)
!$?.success? && out.match?(/error: conflicting types for '[^']+'/)
end
end
def self.with_code(code)
Tempfile.open(['', '.c'], mode: File::BINARY) do |f|
f.puts code
f.close
return yield(f.path)
end
end
private_class_method :with_code
end
if ARGV.size != 3
abort "Usage: #{$0} <c-compiler> <header file> <out>"
end
cc = ARGV[0]
code = File.binread(ARGV[1]) # Current version of the header file.
outfile = ARGV[2]
if MJITHeader.cl_exe?(cc)
cflags = '-DMJIT_HEADER -Zs'
else
cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors'
end
if !MJITHeader.cl_exe?(cc) && !MJITHeader.supported_header?(code)
puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header."
MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile)
exit
end
MJITHeader.remove_predefined_macros!(code)
if MJITHeader.windows? # transformation is broken with Windows headers for now
MJITHeader.remove_harmful_macros!(code)
MJITHeader.check_code!(code, cc, cflags, 'initial')
puts "\nSkipped transforming external functions to static on Windows."
MJITHeader.write(code, outfile)
exit
end
macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW
code_to_check = "#{code}#{macro}" # macro should not affect code again
if MJITHeader.conflicting_types?(code_to_check, cc, cflags)
cflags = "#{cflags} -std=c99" # For AIX gcc
end
# Check initial file correctness in the manner of final output.
MJITHeader.check_code!(code_to_check, cc, cflags, 'initial')
puts "\nTransforming external functions to static:"
stop_pos = -1
extern_names = []
# This loop changes function declarations to static inline.
while (decl_range = MJITHeader.find_decl(code, stop_pos))
stop_pos = decl_range.begin - 1
decl = code[decl_range]
decl_name = MJITHeader.decl_name_of(decl)
if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)
puts "#{PROGRAM}: changing definition of '#{decl_name}' to declaration"
code[decl_range] = decl.sub(/{.+}/m, ';')
elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/)
decl.sub!(/(extern|static|inline) /, ' ')
unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc.
puts "#{PROGRAM}: making declaration of '#{decl_name}' static inline"
end
code[decl_range] = "static inline #{decl}"
elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/
unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP)
puts "#{PROGRAM}: SKIPPED to transform #{decl_name}"
next
end
extern_names << decl_name
decl[match.begin(0)...match.end(0)] = ''
if decl =~ /\bstatic\b/
puts "warning: a static decl inside external definition of '#{decl_name}'"
end
header.sub!(/(extern|inline) /, ' ')
unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc.
puts "#{PROGRAM}: making external definition of '#{decl_name}' static inline"
end
code[decl_range] = "static inline #{header}#{decl}"
end
end
code << macro
# Check the final file correctness
MJITHeader.check_code!(code, cc, cflags, 'final')
MJITHeader.write(code, outfile)
|