* ext/ripper/lib/ripper/tokenizer.rb: new file.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6934 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 35 insertions, 0 deletions

diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
new file mode 100644
index 0000000000..0bd966ea51
--- /dev/null
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -0,0 +1,35 @@
+#
+# ripper/tokenizer.rb
+#
+# Copyright (C) 2004 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper'
+
+class Ripper
+
+  def Ripper.tokenize(str)
+    Tokenizer.tokenize(str)
+  end
+
+  class Tokenizer < ::Ripper
+    def Tokenizer.tokenize(str)
+      new(str).tokenize
+    end
+
+    def tokenize
+      @tokens = []
+      parse
+      @tokens
+    end
+
+    def on__scan(type, tok)
+      @tokens.push tok
+    end
+  end
+
+end