From 2c750e368d28af3a14a4e2ab1f7cb20244a41838 Mon Sep 17 00:00:00 2001
From: aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Mon, 20 Sep 2004 08:42:00 +0000
Subject: * test/ripper/test_scanner_events.rb: tokens must be reordered. *
 ext/ripper/lib/ripper/tokenizer.rb: ditto.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6937 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ChangeLog                          |  6 ++++
 ext/ripper/lib/ripper/tokenizer.rb | 61 +++++++++++++++++++++++++++++++++-----
 test/ripper/test_scanner_events.rb |  7 +++--
 3 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 857bbda54a..adfa489194 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+Mon Sep 20 17:38:43 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* test/ripper/test_scanner_events.rb: tokens must be reordered.
+
+	* ext/ripper/lib/ripper/tokenizer.rb: ditto.
+
 Mon Sep 20 16:58:16 2004  Minero Aoki  <aamine@loveruby.net>
 
 	* parse.y [ripper]: do not delay dispatching.
diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
index 134de9d1e0..e69e5ba519 100644
--- a/ext/ripper/lib/ripper/tokenizer.rb
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -16,22 +16,69 @@ class Ripper
     Tokenizer.tokenize(str)
   end
 
+
   class Tokenizer < ::Ripper
-    def Tokenizer.tokenize(str)
-      new(str).tokenize
+
+    def Tokenizer.tokenize(str, filename = '-', lineno = 1)
+      new(str, filename, lineno).tokenize
+    end
+
+    def initialize(src, filename = '-', lineno = 1)
+      @src = src
+      @__filename = filename
+      @__linestart = lineno
+      @__line = nil
+      @__col = nil
+    end
+
+    def filename
+      @__filename
+    end
+
+    def lineno
+      @__line
+    end
+
+    def column
+      @__col
     end
 
     def tokenize
-      @tokens = []
-      parse
-      @tokens.sort_by {|tok, pos| pos }.map {|tok,| tok }
+      _exec_tokenizer().map {|pos, event, tok| tok }
+    end
+
+    def parse
+      _exec_tokenizer().each do |pos, event, tok|
+        @__line, @__col = *pos
+        on__scan(event, tok)
+        __send__(event, tok)
+      end
+      data
     end
 
     private
 
-    def on__scan(type, tok)
-      @tokens.push [tok, [lineno(),column()]]
+    def _exec_tokenizer
+      TokenSorter.new(@src, @__filename, @__linestart).parse
     end
+
+  end
+
+
+  class TokenSorter < ::Ripper   #:nodoc: internal use only
+
+    def parse
+      @data = []
+      super
+      @data.sort_by {|pos, event, tok| pos }
+    end
+
+    private
+
+    def on__scan(event, tok)
+      @data.push [[lineno(),column()], event, tok]
+    end
+
   end
 
 end
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
index 8a8efbb168..5a2d4e9bf5 100644
--- a/test/ripper/test_scanner_events.rb
+++ b/test/ripper/test_scanner_events.rb
@@ -20,11 +20,11 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
     def parse
       @tokens = []
       super
-      @tokens
+      @tokens.sort_by {|tok,pos| pos }.map {|tok,| tok }
     end
 
     def on__scan(type, tok)
-      @tokens.push tok if !@target or type == @target
+      @tokens.push [tok,[lineno(),column()]] if !@target or type == @target
     end
   end
 
@@ -80,7 +80,8 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
   def validate_location(src)
     data = PosInfo.new(src).parse
     buf = ''
-    data.each do |tok, type, line, col|
+    data.sort_by {|tok, type, line, col| [line,col] }\
+        .each do |tok, type, line, col|
       assert_equal buf.count("\n") + 1, line,
           "wrong lineno: #{tok.inspect} (#{type}) [#{line}:#{col}]"
       assert_equal buf.sub(/\A.*\n/m, '').size, col,
-- 
cgit v1.2.3