blob: 0bd966ea515d61a0efa43dfa576579d1470b6dca (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
#
# ripper/tokenizer.rb
#
# Copyright (C) 2004 Minero Aoki
#
# This program is free software.
# You can distribute and/or modify this program under the Ruby License.
# For details of Ruby License, see ruby/COPYING.
#
require 'ripper'
class Ripper
def Ripper.tokenize(str)
Tokenizer.tokenize(str)
end
class Tokenizer < ::Ripper
def Tokenizer.tokenize(str)
new(str).tokenize
end
def tokenize
@tokens = []
parse
@tokens
end
def on__scan(type, tok)
@tokens.push tok
end
end
end
|