diff options
-rw-r--r-- | ChangeLog | 4 | ||||
-rw-r--r-- | ext/ripper/lib/ripper/tokenizer.rb | 35 |
2 files changed, 39 insertions, 0 deletions
@@ -1,3 +1,7 @@ +Mon Sep 20 15:17:47 2004 Minero Aoki <aamine@loveruby.net> + + * ext/ripper/lib/ripper/tokenizer.rb: new file. + Mon Sep 20 15:13:52 2004 Minero Aoki <aamine@loveruby.net> * test/ripper/test_scanner_events.rb: test #lineno and #column. diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb new file mode 100644 index 0000000000..0bd966ea51 --- /dev/null +++ b/ext/ripper/lib/ripper/tokenizer.rb @@ -0,0 +1,35 @@ +# +# ripper/tokenizer.rb +# +# Copyright (C) 2004 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper' + +class Ripper + + def Ripper.tokenize(str) + Tokenizer.tokenize(str) + end + + class Tokenizer < ::Ripper + def Tokenizer.tokenize(str) + new(str).tokenize + end + + def tokenize + @tokens = [] + parse + @tokens + end + + def on__scan(type, tok) + @tokens.push tok + end + end + +end |