diff options
-rw-r--r-- | ext/ripper/lib/ripper/lexer.rb | 9 | ||||
-rw-r--r-- | test/ripper/test_lexer.rb | 11 | ||||
-rw-r--r-- | test/ripper/test_scanner_events.rb | 1 |
3 files changed, 20 insertions, 1 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb index 350b513c25..63dcf85132 100644 --- a/ext/ripper/lib/ripper/lexer.rb +++ b/ext/ripper/lib/ripper/lexer.rb @@ -66,13 +66,20 @@ class Ripper private def on_heredoc_dedent(v, w) - @buf.last.each do |e| + ignored_sp = [] + heredoc = @buf.last + heredoc.each_with_index do |e, i| if Elem === e and e.event == :on_tstring_content + tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok if (n = dedent_string(e.tok, w)) > 0 + ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])] e.pos[1] += n end end end + ignored_sp.reverse_each do |i, e| + heredoc[i, 0] = [e] + end v end diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb index 2ec66cd592..72b7f1b5fd 100644 --- a/test/ripper/test_lexer.rb +++ b/test/ripper/test_lexer.rb @@ -20,5 +20,16 @@ class TestRipper::Lexer < Test::Unit::TestCase D E assert_equal(str, Ripper.tokenize(str).join(""), bug) + + str = <<~'E' + <<~"D" + #{ + <<~"B" + this must be a valid ruby + B + } + D + E + assert_equal(str, Ripper.tokenize(str).join(""), bug) end end diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index 027d56d1e6..ef49fc18ba 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -105,6 +105,7 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri") assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS"], [[1, 6], :on_nl, "\n"], + [[2, 0], :on_ignored_sp, " "], [[2, 2], :on_tstring_content, "heredoc\n"], [[3, 0], :on_heredoc_end, "EOS"] ], |