1 files changed, 108 insertions, 0 deletions
diff --git a/lib/rubygems/request_set/lockfile/tokenizer.rb b/lib/rubygems/request_set/lockfile/tokenizer.rb
new file mode 100644
index 0000000000..73c9a834bb
--- /dev/null
+++ b/lib/rubygems/request_set/lockfile/tokenizer.rb
@@ -0,0 +1,108 @@
+require 'strscan'
+require 'rubygems/request_set/lockfile/parser'
+
+class Gem::RequestSet::Lockfile::Tokenizer
+  def self.from_file file
+    new File.read(file), file
+  end
+
+  def initialize input, filename = nil, line = 0, pos = 0
+    @line     = line
+    @line_pos = pos
+    @tokens   = []
+    @filename = filename
+    tokenize input
+  end
+
+  def make_parser set, platforms
+    Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename
+  end
+
+  def to_a
+    @tokens
+  end
+
+  def skip type
+    @tokens.shift while not @tokens.empty? and peek.first == type
+  end
+
+  ##
+  # Calculates the column (by byte) and the line of the current token based on
+  # +byte_offset+.
+
+  def token_pos byte_offset # :nodoc:
+    [byte_offset - @line_pos, @line]
+  end
+
+  def empty?
+    @tokens.empty?
+  end
+
+  def unshift token
+    @tokens.unshift token
+  end
+
+  def next_token
+    @tokens.shift
+  end
+  alias :shift :next_token
+
+  def peek
+    @tokens.first || [:EOF]
+  end
+
+  private
+
+  def tokenize input
+    s = StringScanner.new input
+
+    until s.eos? do
+      pos = s.pos
+
+      pos = s.pos if leading_whitespace = s.scan(/ +/)
+
+      if s.scan(/[<|=>]{7}/) then
+        message = "your #{@filename} contains merge conflict markers"
+        column, line = token_pos pos
+
+        raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
+      end
+
+      @tokens <<
+        case
+        when s.scan(/\r?\n/) then
+          token = [:newline, nil, *token_pos(pos)]
+          @line_pos = s.pos
+          @line += 1
+          token
+        when s.scan(/[A-Z]+/) then
+          if leading_whitespace then
+            text = s.matched
+            text += s.scan(/[^\s)]*/).to_s # in case of no match
+            [:text, text, *token_pos(pos)]
+          else
+            [:section, s.matched, *token_pos(pos)]
+          end
+        when s.scan(/([a-z]+):\s/) then
+          s.pos -= 1 # rewind for possible newline
+          [:entry, s[1], *token_pos(pos)]
+        when s.scan(/\(/) then
+          [:l_paren, nil, *token_pos(pos)]
+        when s.scan(/\)/) then
+          [:r_paren, nil, *token_pos(pos)]
+        when s.scan(/<=|>=|=|~>|<|>|!=/) then
+          [:requirement, s.matched, *token_pos(pos)]
+        when s.scan(/,/) then
+          [:comma, nil, *token_pos(pos)]
+        when s.scan(/!/) then
+          [:bang, nil, *token_pos(pos)]
+        when s.scan(/[^\s),!]*/) then
+          [:text, s.matched, *token_pos(pos)]
+        else
+          raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
+        end
+    end
+
+    @tokens
+  end
+end