diff options
-rw-r--r-- | ChangeLog | 8 | ||||
-rw-r--r-- | parse.y | 13 | ||||
-rw-r--r-- | test/ripper/test_scanner_events.rb | 17 |
3 files changed, 36 insertions, 2 deletions
@@ -1,3 +1,11 @@ +Mon Jun 13 23:38:23 2011 Nobuyoshi Nakada <nobu@ruby-lang.org> + + * parse.y (parser_parse_string): flush delayed token. based on a + patch by Masaya Tarui in [ruby-dev:43762]. Bug #4544 + + * parse.y (yylex): revert r24557. delayed token at the end of + string should be flushed already by the above change. + Mon Jun 13 23:33:31 2011 Nobuyoshi Nakada <nobu@ruby-lang.org> * ext/io/console/console.c (console_dev): console should be @@ -5984,6 +5984,18 @@ parser_parse_string(struct parser_params *parser, NODE *quote) tokfix(); set_yylval_str(STR_NEW3(tok(), toklen(), enc, func)); + +#ifdef RIPPER + if (!NIL_P(parser->delayed)){ + ptrdiff_t len = lex_p - parser->tokp; + if (len > 0) { + rb_enc_str_buf_cat(parser->delayed, parser->tokp, len, enc); + } + ripper_dispatch_delayed_token(parser, tSTRING_CONTENT); + parser->tokp = lex_p; + } +#endif + return tSTRING_CONTENT; } @@ -7853,6 +7865,7 @@ yylex(void *p) #ifdef RIPPER if (!NIL_P(parser->delayed)) { ripper_dispatch_delayed_token(parser, t); + return t; } if (t != 0) ripper_dispatch_scan_event(parser, t); diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index 25e4b13e6c..9005e46f4e 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -67,10 +67,17 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase [[3, 0], :on_heredoc_end, "EOS"]], Ripper.lex("<<EOS\nheredoc\nEOS") assert_equal [[[1, 0], :on_regexp_beg, "/"], - [[1, 1], :on_tstring_content, "foo\n"], - [[2, 0], :on_tstring_content, "bar"], + [[1, 1], :on_tstring_content, "foo\nbar"], [[2, 3], :on_regexp_end, "/"]], Ripper.lex("/foo\nbar/") + assert_equal [[[1, 0], :on_regexp_beg, "/"], + [[1, 1], :on_tstring_content, "foo\n\u3020"], + [[2, 3], :on_regexp_end, "/"]], + Ripper.lex("/foo\n\u3020/") + assert_equal [[[1, 0], :on_tstring_beg, "'"], + [[1, 1], :on_tstring_content, "foo\n\xe3\x80\xa0"], + [[2, 3], :on_tstring_end, "'"]], + Ripper.lex("'foo\n\xe3\x80\xa0'") end def test_location @@ -534,6 +541,12 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase scan('tstring_content', '"abc#{1}def"') assert_equal ['sym'], scan('tstring_content', ':"sym"') + assert_equal ['a b c'], + scan('tstring_content', ':"a b c"'), + "bug#4544" + assert_equal ["a\nb\nc"], + scan('tstring_content', ":'a\nb\nc'"), + "bug#4544" end def test_tstring_end |