diff options
author | nobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-11-23 04:30:23 +0000 |
---|---|---|
committer | nobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-11-23 04:30:23 +0000 |
commit | dee6a910024e119b6064031487f87d927d960304 (patch) | |
tree | 607b3cc0e2c6c9049ea53a5d54c51fa61a1e4379 | |
parent | 717b7fd9c45d50a3fb24e744f61628bc2f5d4b75 (diff) | |
download | ruby-dee6a910024e119b6064031487f87d927d960304.tar.gz |
ripper.y: fix word list events
* parse.y (parser_skip_words_sep): QWORDS_BEG should not include
the first separators in ripper.
* parse.y (parser_parse_string): WORDS_SEP should not include
the closing parentheses of a word list in ripper, should include
spaces at beginning of lines. [ruby-core:83864] [Bug #14126]
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60883 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r-- | parse.y | 70 | ||||
-rw-r--r-- | test/ripper/test_scanner_events.rb | 27 |
2 files changed, 64 insertions, 33 deletions
@@ -5819,6 +5819,26 @@ parser_str_new(const char *p, long n, rb_encoding *enc, int func, rb_encoding *e #define peekc() peekc_n(0) #define peekc_n(n) (lex_p+(n) < lex_pend ? (unsigned char)lex_p[n] : -1) +#ifdef RIPPER +static void +parser_add_delayed_token(struct parser_params *parser, const char *tok, const char *end) +{ + if (tok < end) { + if (!has_delayed_token()) { + parser->delayed = rb_str_buf_new(1024); + rb_enc_associate(parser->delayed, current_enc); + parser->delayed_line = ruby_sourceline; + parser->delayed_col = (int)(tok - lex_pbeg); + } + rb_str_buf_cat(parser->delayed, tok, end - tok); + parser->tokp = end; + } +} +#define add_delayed_token(tok, end) parser_add_delayed_token(parser, (tok), (end)) +#else +#define add_delayed_token(tok, end) ((void)(tok), (void)(end)) +#endif + static int parser_nextline(struct parser_params *parser) { @@ -5835,22 +5855,7 @@ parser_nextline(struct parser_params *parser) } parser->cr_seen = FALSE; } -#ifdef RIPPER - if (parser->tokp < lex_pend) { - if (!has_delayed_token()) { - parser->delayed = rb_str_buf_new(1024); - rb_enc_associate(parser->delayed, current_enc); - rb_str_buf_cat(parser->delayed, - parser->tokp, lex_pend - parser->tokp); - parser->delayed_line = ruby_sourceline; - parser->delayed_col = (int)(parser->tokp - lex_pbeg); - } - else { - rb_str_buf_cat(parser->delayed, - parser->tokp, lex_pend - parser->tokp); - } - } -#endif + add_delayed_token(parser->tokp, lex_pend); if (heredoc_end > 0) { ruby_sourceline = heredoc_end; heredoc_end = 0; @@ -6597,6 +6602,9 @@ parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote) VALUE lit; if (func & STR_FUNC_TERM) { +#ifdef RIPPER + if (func & STR_FUNC_QWORDS) nextc(); /* delayed term */ +#endif SET_LEX_STATE(EXPR_END|EXPR_ENDARG); lex_strterm = 0; return func & STR_FUNC_REGEXP ? tREGEXP_END : tSTRING_END; @@ -6609,12 +6617,17 @@ parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote) if (c == term && !quote->u0.nest) { if (func & STR_FUNC_QWORDS) { quote->u1.func |= STR_FUNC_TERM; +#ifdef RIPPER + pushback(c); /* dispatch the term at tSTRING_END */ +#endif + add_delayed_token(parser->tokp, lex_p); return ' '; } return parser_string_term(parser, func); } if (space) { pushback(c); + add_delayed_token(parser->tokp, lex_p); return ' '; } newtok(); @@ -7853,6 +7866,19 @@ parse_qmark(struct parser_params *parser, int space_seen) return tCHAR; } +#ifndef RIPPER +static void +parser_skip_words_sep(struct parser_params *parser) +{ + int c; + do {c = nextc();} while (ISSPACE(c)); + pushback(c); +} +#define skip_words_sep() parser_skip_words_sep(parser) +#else +#define skip_words_sep() ((void)0) +#endif + static enum yytokentype parse_percent(struct parser_params *parser, const int space_seen, const enum lex_state_e last_state) { @@ -7897,26 +7923,22 @@ parse_percent(struct parser_params *parser, const int space_seen, const enum lex case 'W': lex_strterm = NEW_STRTERM(str_dword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tWORDS_BEG; case 'w': lex_strterm = NEW_STRTERM(str_sword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tQWORDS_BEG; case 'I': lex_strterm = NEW_STRTERM(str_dword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tSYMBOLS_BEG; case 'i': lex_strterm = NEW_STRTERM(str_sword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tQSYMBOLS_BEG; case 'x': diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index 90fd599a31..b7c5a02322 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -637,8 +637,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase scan('words_beg', '%W()') assert_equal ['%W('], scan('words_beg', '%W(w w w)') - assert_equal ['%W( '], + assert_equal ['%W('], scan('words_beg', '%W( w w w )') + assert_equal ['%W('], + scan('words_beg', "%W(\nw)") end def test_qwords_beg @@ -648,8 +650,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase scan('qwords_beg', '%w()') assert_equal ['%w('], scan('qwords_beg', '%w(w w w)') - assert_equal ['%w( '], + assert_equal ['%w('], scan('qwords_beg', '%w( w w w )') + assert_equal ['%w('], + scan('qwords_beg', "%w(\nw)") end def test_qsymbols_beg @@ -659,8 +663,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase scan('qsymbols_beg', '%i()') assert_equal ['%i('], scan('qsymbols_beg', '%i(w w w)') - assert_equal ['%i( '], + assert_equal ['%i('], scan('qsymbols_beg', '%i( w w w )') + assert_equal ['%i('], + scan('qsymbols_beg', "%i(\nw)") end def test_symbols_beg @@ -670,22 +676,25 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase scan('symbols_beg', '%I()') assert_equal ['%I('], scan('symbols_beg', '%I(w w w)') - assert_equal ['%I( '], + assert_equal ['%I('], scan('symbols_beg', '%I( w w w )') + assert_equal ['%I('], + scan('symbols_beg', "%I(\nw)") end - # FIXME: Close paren must not present (`words_end' scanner event?). def test_words_sep assert_equal [], scan('words_sep', '') - assert_equal [')'], + assert_equal [], scan('words_sep', '%w()') - assert_equal [' ', ' ', ')'], + assert_equal [' ', ' '], scan('words_sep', '%w(w w w)') - assert_equal [' ', ' ', ' )'], + assert_equal [' ', ' ', ' ', ' '], scan('words_sep', '%w( w w w )') - assert_equal ["\n", ' ', ' )'], + assert_equal [' ', "\n", ' ', ' '], scan('words_sep', "%w( w\nw w )") + assert_equal ["\n\n", "\n ", ' ', ' '], + scan('words_sep', "%w(\n\nw\n w w )") end def test_heredoc_beg |