aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNobuyoshi Nakada <nobu@ruby-lang.org>2019-05-29 16:02:09 +0900
committerNobuyoshi Nakada <nobu@ruby-lang.org>2019-05-29 18:21:28 +0900
commit5ceff480c2c1696e9893a1e5e09e39f8425c6c5a (patch)
tree877283a4b6f0364cb3d81110a38b0239092c78ba
parent7c0639f3f83f85557aff87e94da1afd373555bcb (diff)
downloadruby-5ceff480c2c1696e9893a1e5e09e39f8425c6c5a.tar.gz
ripper: Ripper::Lexer#scan
* ext/ripper/lib/ripper/lexer.rb (Ripper::Lexer#scan): parses the code and returns the result elements including errors. [EXPERIMENTAL]
-rw-r--r--ext/ripper/lib/ripper/lexer.rb33
-rw-r--r--lib/irb/color.rb24
2 files changed, 39 insertions, 18 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index a5ba5b6890..d1d257582d 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -60,13 +60,13 @@ class Ripper
def nobits?(i) to_int.nobits?(i) end
end
- Elem = Struct.new(:pos, :event, :tok, :state) do
- def initialize(pos, event, tok, state)
- super(pos, event, tok, State.new(state))
+ Elem = Struct.new(:pos, :event, :tok, :state, :message) do
+ def initialize(pos, event, tok, state, message = nil)
+ super(pos, event, tok, State.new(state), message)
end
def inspect
- "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}>"
+ "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
end
def pretty_print(q)
@@ -76,10 +76,23 @@ class Ripper
q.breakable
q.text("token: ")
tok.pretty_print(q)
+ if message
+ q.breakable
+ q.text("message: ")
+ q.text(message)
+ end
}
end
+
+ def to_a
+ a = super
+ a.pop unless a.last
+ a
+ end
end
+ attr_reader :errors
+
def tokenize
parse().sort_by(&:pos).map(&:tok)
end
@@ -88,7 +101,13 @@ class Ripper
parse().sort_by(&:pos).map(&:to_a)
end
+ # parse the code and returns elements including errors.
+ def scan
+ (parse() + errors).sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
+ end
+
def parse
+ @errors = []
@buf = []
@stack = []
super
@@ -144,6 +163,12 @@ class Ripper
@buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end
+ def on_error(mesg)
+ @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+ end
+ alias on_parse_error on_error
+ alias compile_error on_error
+
(SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
alias_method event, :_push_token
end
diff --git a/lib/irb/color.rb b/lib/irb/color.rb
index b1f7c21743..71557ffbb9 100644
--- a/lib/irb/color.rb
+++ b/lib/irb/color.rb
@@ -56,7 +56,8 @@ module IRB # :nodoc:
on_tstring_content: [[RED], ALL],
on_tstring_end: [[RED], ALL],
on_words_beg: [[RED], ALL],
- ERROR: [[RED, REVERSE], ALL],
+ on_parse_error: [[RED, REVERSE], ALL],
+ compile_error: [[RED, REVERSE], ALL],
}
rescue NameError
# Give up highlighting Ripper-incompatible older Ruby
@@ -64,17 +65,6 @@ module IRB # :nodoc:
end
private_constant :TOKEN_SEQ_EXPRS
- class Lexer < Ripper::Lexer
- if method_defined?(:token)
- def on_error(mesg)
- # :ERROR comes before other :on_ symbols
- @buf.push Elem.new([lineno(), column()], :ERROR, token(), state())
- end
- alias on_parse_error on_error
- alias compile_error on_error
- end
- end
-
class << self
def colorable?
$stdout.tty? && (/mswin|mingw/ =~ RUBY_PLATFORM || (ENV.key?('TERM') && ENV['TERM'] != 'dumb'))
@@ -107,6 +97,10 @@ module IRB # :nodoc:
"#{seq.map { |s| "\e[#{const_get(s)}m" }.join('')}#{text}#{clear}"
end
+ def scan(code)
+ Ripper::Lexer.new(code).scan
+ end
+
def colorize_code(code)
return code unless colorable?
@@ -114,7 +108,7 @@ module IRB # :nodoc:
colored = +''
length = 0
- Lexer.new(code).parse.sort_by(&:pos).each do |elem|
+ scan(code).each do |elem|
token = elem.event
str = elem.tok
expr = elem.state
@@ -139,7 +133,9 @@ module IRB # :nodoc:
private
def dispatch_seq(token, expr, str, in_symbol:)
- if in_symbol
+ if token == :on_parse_error or token == :compile_error
+ TOKEN_SEQ_EXPRS[token][0]
+ elsif in_symbol
[YELLOW]
elsif TOKEN_KEYWORDS.fetch(token, []).include?(str)
[CYAN, BOLD]