diff options
Diffstat (limited to 'lib/rubygems/request_set/lockfile/tokenizer.rb')
-rw-r--r-- | lib/rubygems/request_set/lockfile/tokenizer.rb | 108 |
1 files changed, 108 insertions, 0 deletions
diff --git a/lib/rubygems/request_set/lockfile/tokenizer.rb b/lib/rubygems/request_set/lockfile/tokenizer.rb new file mode 100644 index 0000000000..73c9a834bb --- /dev/null +++ b/lib/rubygems/request_set/lockfile/tokenizer.rb @@ -0,0 +1,108 @@ +require 'strscan' +require 'rubygems/request_set/lockfile/parser' + +class Gem::RequestSet::Lockfile::Tokenizer + def self.from_file file + new File.read(file), file + end + + def initialize input, filename = nil, line = 0, pos = 0 + @line = line + @line_pos = pos + @tokens = [] + @filename = filename + tokenize input + end + + def make_parser set, platforms + Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename + end + + def to_a + @tokens + end + + def skip type + @tokens.shift while not @tokens.empty? and peek.first == type + end + + ## + # Calculates the column (by byte) and the line of the current token based on + # +byte_offset+. + + def token_pos byte_offset # :nodoc: + [byte_offset - @line_pos, @line] + end + + def empty? + @tokens.empty? + end + + def unshift token + @tokens.unshift token + end + + def next_token + @tokens.shift + end + alias :shift :next_token + + def peek + @tokens.first || [:EOF] + end + + private + + def tokenize input + s = StringScanner.new input + + until s.eos? do + pos = s.pos + + pos = s.pos if leading_whitespace = s.scan(/ +/) + + if s.scan(/[<|=>]{7}/) then + message = "your #{@filename} contains merge conflict markers" + column, line = token_pos pos + + raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename + end + + @tokens << + case + when s.scan(/\r?\n/) then + token = [:newline, nil, *token_pos(pos)] + @line_pos = s.pos + @line += 1 + token + when s.scan(/[A-Z]+/) then + if leading_whitespace then + text = s.matched + text += s.scan(/[^\s)]*/).to_s # in case of no match + [:text, text, *token_pos(pos)] + else + [:section, s.matched, *token_pos(pos)] + end + when s.scan(/([a-z]+):\s/) then + s.pos -= 1 # rewind for possible newline + [:entry, s[1], *token_pos(pos)] + when s.scan(/\(/) then + [:l_paren, nil, *token_pos(pos)] + when s.scan(/\)/) then + [:r_paren, nil, *token_pos(pos)] + when s.scan(/<=|>=|=|~>|<|>|!=/) then + [:requirement, s.matched, *token_pos(pos)] + when s.scan(/,/) then + [:comma, nil, *token_pos(pos)] + when s.scan(/!/) then + [:bang, nil, *token_pos(pos)] + when s.scan(/[^\s),!]*/) then + [:text, s.matched, *token_pos(pos)] + else + raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}" + end + end + + @tokens + end +end |