diff options
| -rw-r--r-- | ChangeLog | 6 | ||||
| -rw-r--r-- | ext/ripper/lib/ripper/tokenizer.rb | 61 | ||||
| -rw-r--r-- | test/ripper/test_scanner_events.rb | 7 |
3 files changed, 64 insertions, 10 deletions
@@ -1,3 +1,9 @@ +Mon Sep 20 17:38:43 2004 Minero Aoki <aamine@loveruby.net> + + * test/ripper/test_scanner_events.rb: tokens must be reordered. + + * ext/ripper/lib/ripper/tokenizer.rb: ditto. + Mon Sep 20 16:58:16 2004 Minero Aoki <aamine@loveruby.net> * parse.y [ripper]: do not delay dispatching. diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb index 134de9d1e0..e69e5ba519 100644 --- a/ext/ripper/lib/ripper/tokenizer.rb +++ b/ext/ripper/lib/ripper/tokenizer.rb @@ -16,22 +16,69 @@ class Ripper Tokenizer.tokenize(str) end + class Tokenizer < ::Ripper - def Tokenizer.tokenize(str) - new(str).tokenize + + def Tokenizer.tokenize(str, filename = '-', lineno = 1) + new(str, filename, lineno).tokenize + end + + def initialize(src, filename = '-', lineno = 1) + @src = src + @__filename = filename + @__linestart = lineno + @__line = nil + @__col = nil + end + + def filename + @__filename + end + + def lineno + @__line + end + + def column + @__col end def tokenize - @tokens = [] - parse - @tokens.sort_by {|tok, pos| pos }.map {|tok,| tok } + _exec_tokenizer().map {|pos, event, tok| tok } + end + + def parse + _exec_tokenizer().each do |pos, event, tok| + @__line, @__col = *pos + on__scan(event, tok) + __send__(event, tok) + end + data end private - def on__scan(type, tok) - @tokens.push [tok, [lineno(),column()]] + def _exec_tokenizer + TokenSorter.new(@src, @__filename, @__linestart).parse end + + end + + + class TokenSorter < ::Ripper #:nodoc: internal use only + + def parse + @data = [] + super + @data.sort_by {|pos, event, tok| pos } + end + + private + + def on__scan(event, tok) + @data.push [[lineno(),column()], event, tok] + end + end end diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index 8a8efbb168..5a2d4e9bf5 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -20,11 +20,11 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase def parse @tokens = [] super - @tokens + @tokens.sort_by {|tok,pos| pos }.map {|tok,| tok } end def on__scan(type, tok) - @tokens.push tok if !@target or type == @target + @tokens.push [tok,[lineno(),column()]] if !@target or type == @target end end @@ -80,7 +80,8 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase def validate_location(src) data = PosInfo.new(src).parse buf = '' - data.each do |tok, type, line, col| + data.sort_by {|tok, type, line, col| [line,col] }\ + .each do |tok, type, line, col| assert_equal buf.count("\n") + 1, line, "wrong lineno: #{tok.inspect} (#{type}) [#{line}:#{col}]" assert_equal buf.sub(/\A.*\n/m, '').size, col, |