diff options
Diffstat (limited to 'ext/ripper')
-rw-r--r-- | ext/ripper/lib/ripper/tokenizer.rb | 35 |
1 files changed, 35 insertions, 0 deletions
diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb new file mode 100644 index 0000000000..0bd966ea51 --- /dev/null +++ b/ext/ripper/lib/ripper/tokenizer.rb @@ -0,0 +1,35 @@ +# +# ripper/tokenizer.rb +# +# Copyright (C) 2004 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper' + +class Ripper + + def Ripper.tokenize(str) + Tokenizer.tokenize(str) + end + + class Tokenizer < ::Ripper + def Tokenizer.tokenize(str) + new(str).tokenize + end + + def tokenize + @tokens = [] + parse + @tokens + end + + def on__scan(type, tok) + @tokens.push tok + end + end + +end |