From db8803d583c31767a2f6771ecdf929bf5ee6c278 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 1 Nov 2023 13:29:39 -0400 Subject: [ruby/prism] More consistent lex modes with %q https://github.com/ruby/prism/commit/014f714ed1 --- prism/prism.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/prism/prism.c b/prism/prism.c index 132cf3a97d..c98062f40a 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -275,6 +275,16 @@ lex_mode_push_list(pm_parser_t *parser, bool interpolation, uint8_t delimiter) { return lex_mode_push(parser, lex_mode); } +/** + * Push on a new list lex mode that is only used for compatibility. This is + * called when we're at the end of the file. We want the parser to be able to + * perform its normal error tolerance. + */ +static inline bool +lex_mode_push_list_eof(pm_parser_t *parser) { + return lex_mode_push_list(parser, false, '\0'); +} + /** * Push on a new regexp lex mode. */ @@ -346,6 +356,16 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed return lex_mode_push(parser, lex_mode); } +/** + * Push on a new string lex mode that is only used for compatibility. This is + * called when we're at the end of the file. We want the parser to be able to + * perform its normal error tolerance. + */ +static inline bool +lex_mode_push_string_eof(pm_parser_t *parser) { + return lex_mode_push_string(parser, false, false, '\0', '\0'); +} + /** * Pop the current lex state off the stack. If we're within the pre-allocated * space of the lex state stack, then we'll just decrement the index. Otherwise @@ -8791,6 +8811,8 @@ parser_lex(pm_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_list(parser, false, *parser->current.end++); + } else { + lex_mode_push_list_eof(parser); } LEX(PM_TOKEN_PERCENT_LOWER_I); @@ -8800,6 +8822,8 @@ parser_lex(pm_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_list(parser, true, *parser->current.end++); + } else { + lex_mode_push_list_eof(parser); } LEX(PM_TOKEN_PERCENT_UPPER_I); @@ -8811,6 +8835,8 @@ parser_lex(pm_parser_t *parser) { lex_mode_push_regexp(parser, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); pm_newline_list_check_append(&parser->newline_list, parser->current.end); parser->current.end++; + } else { + lex_mode_push_regexp(parser, '\0', '\0'); } LEX(PM_TOKEN_REGEXP_BEGIN); @@ -8822,6 +8848,8 @@ parser_lex(pm_parser_t *parser) { lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); pm_newline_list_check_append(&parser->newline_list, parser->current.end); parser->current.end++; + } else { + lex_mode_push_string_eof(parser); } LEX(PM_TOKEN_STRING_BEGIN); @@ -8833,6 +8861,8 @@ parser_lex(pm_parser_t *parser) { lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); pm_newline_list_check_append(&parser->newline_list, parser->current.end); parser->current.end++; + } else { + lex_mode_push_string_eof(parser); } LEX(PM_TOKEN_STRING_BEGIN); @@ -8844,6 +8874,8 @@ parser_lex(pm_parser_t *parser) { lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM); parser->current.end++; + } else { + lex_mode_push_string_eof(parser); } LEX(PM_TOKEN_SYMBOL_BEGIN); @@ -8853,6 +8885,8 @@ parser_lex(pm_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_list(parser, false, *parser->current.end++); + } else { + lex_mode_push_list_eof(parser); } LEX(PM_TOKEN_PERCENT_LOWER_W); @@ -8862,6 +8896,8 @@ parser_lex(pm_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_list(parser, true, *parser->current.end++); + } else { + lex_mode_push_list_eof(parser); } LEX(PM_TOKEN_PERCENT_UPPER_W); @@ -8872,6 +8908,8 @@ parser_lex(pm_parser_t *parser) { if (parser->current.end < parser->end) { lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); parser->current.end++; + } else { + lex_mode_push_string_eof(parser); } LEX(PM_TOKEN_PERCENT_LOWER_X); -- cgit v1.2.3