diff options
| -rw-r--r-- | parse.y | 8 | ||||
| -rw-r--r-- | test/ripper/test_lexer.rb | 61 |
2 files changed, 69 insertions, 0 deletions
@@ -7631,6 +7631,14 @@ add_delayed_token(struct parser_params *p, const char *tok, const char *end, int #endif if (tok < end) { + if (has_delayed_token(p)) { + bool next_line = end_with_newline_p(p, p->delayed.token); + int end_line = (next_line ? 1 : 0) + p->delayed.end_line; + int end_col = (next_line ? 0 : p->delayed.end_col); + if (end_line != p->ruby_sourceline || end_col != tok - p->lex.pbeg) { + dispatch_delayed_token(p, tSTRING_CONTENT); + } + } if (!has_delayed_token(p)) { p->delayed.token = rb_str_buf_new(end - tok); rb_enc_associate(p->delayed.token, p->enc); diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb index 92587ae4c0..7bff858705 100644 --- a/test/ripper/test_lexer.rb +++ b/test/ripper/test_lexer.rb @@ -273,4 +273,65 @@ world" assert_include(Ripper.tokenize(code).join(""), "+1") end + + def test_nested_heredoc + code = <<~'HEREDOC' + <<~H1 + 1 + #{<<~H2} + 2 + H2 + 3 + H1 + HEREDOC + + expected = [ + [[1, 0], :on_heredoc_beg, "<<~H1", state(:EXPR_BEG)], + [[1, 5], :on_nl, "\n", state(:EXPR_BEG)], + [[2, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[2, 2], :on_tstring_content, "1\n", state(:EXPR_BEG)], + [[3, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[3, 2], :on_embexpr_beg, "\#{", state(:EXPR_BEG)], + [[3, 4], :on_heredoc_beg, "<<~H2", state(:EXPR_BEG)], + [[3, 9], :on_embexpr_end, "}", state(:EXPR_END)], + [[3, 10], :on_tstring_content, "\n", state(:EXPR_BEG)], + [[4, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[4, 4], :on_tstring_content, "2\n", state(:EXPR_BEG)], + [[5, 0], :on_heredoc_end, " H2\n", state(:EXPR_BEG)], + [[6, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[6, 2], :on_tstring_content, "3\n", state(:EXPR_BEG)], + [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)], + ] + assert_equal(code, Ripper.tokenize(code).join("")) + assert_equal(expected, result = Ripper.lex(code), + proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}}) + + code = <<~'HEREDOC' + <<-H1 + 1 + #{<<~H2} + 2 + H2 + 3 + H1 + HEREDOC + + expected = [ + [[1, 0], :on_heredoc_beg, "<<-H1", state(:EXPR_BEG)], + [[1, 5], :on_nl, "\n", state(:EXPR_BEG)], + [[2, 0], :on_tstring_content, " 1\n ", state(:EXPR_BEG)], + [[3, 2], :on_embexpr_beg, "\#{", state(:EXPR_BEG)], + [[3, 4], :on_heredoc_beg, "<<~H2", state(:EXPR_BEG)], + [[3, 9], :on_embexpr_end, "}", state(:EXPR_END)], + [[3, 10], :on_tstring_content, "\n", state(:EXPR_BEG)], + [[4, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[4, 4], :on_tstring_content, "2\n", state(:EXPR_BEG)], + [[5, 0], :on_heredoc_end, " H2\n", state(:EXPR_BEG)], + [[6, 0], :on_tstring_content, " 3\n", state(:EXPR_BEG)], + [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)], + ] + assert_equal(code, Ripper.tokenize(code).join("")) + assert_equal(expected, result = Ripper.lex(code), + proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}}) + end end |