1 files changed, 33 insertions, 5 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 586d0807a1..e0460e8b02 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -44,28 +44,56 @@ class Ripper
   end
 
   class Lexer < ::Ripper   #:nodoc: internal use only
+    Elem = Struct.new(:pos, :event, :tok)
+
     def tokenize
-      lex().map {|pos, event, tok| tok }
+      parse().sort_by(&:pos).map(&:tok)
     end
 
     def lex
-      parse().sort_by {|pos, event, tok| pos }
+      parse().sort_by(&:pos).map(&:to_a)
     end
 
     def parse
       @buf = []
+      @stack = []
       super
+      @buf.flatten!
       @buf
     end
 
     private
 
+    def on_heredoc_dedent(v, w)
+      @buf.each do |e|
+        if e.event == :on_tstring_content
+          if (n = dedent_string(e.tok, w)) > 0
+            e.pos[1] += n
+          end
+        end
+      end
+      v
+    end
+
+    def on_heredoc_beg(tok)
+      @stack.push @buf
+      buf = []
+      @buf << buf
+      @buf = buf
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+    end
+
+    def on_heredoc_end(tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf = @stack.pop
+    end
+
     def _push_token(tok)
-      @buf.push [[lineno(), column()], __callee__, tok]
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
     end
 
-    SCANNER_EVENTS.each do |event|
-      alias_method "on_#{event}", :_push_token
+    (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
+      alias_method event, :_push_token
     end
   end