diff options
author | mame <mame@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-11-04 07:21:36 +0000 |
---|---|---|
committer | mame <mame@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-11-04 07:21:36 +0000 |
commit | 12697c0867485214d4239b7dcfe57b7537eb835c (patch) | |
tree | 7b19106a1283fcb3052b59b372a8f414822c12ba /parse.y | |
parent | 51423bd9ce1e7ec1404045a8f9fec4cd3466cfc2 (diff) | |
download | ruby-12697c0867485214d4239b7dcfe57b7537eb835c.tar.gz |
Revert "Revert "Replace NODE_STRTERM and NODE_HEREDOC with imemo_strterm""
Retry r60634
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60637 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'parse.y')
-rw-r--r-- | parse.y | 131 |
1 files changed, 79 insertions, 52 deletions
@@ -178,6 +178,8 @@ typedef struct token_info { struct token_info *next; } token_info; +typedef struct rb_strterm_struct rb_strterm_t; + /* Structure of Lexer Buffer: @@ -193,7 +195,7 @@ struct parser_params { YYSTYPE *lval; struct { - NODE *strterm; + rb_strterm_t *strterm; VALUE (*gets)(struct parser_params*,VALUE); VALUE input; VALUE lastline; @@ -738,17 +740,41 @@ static int lvar_defined_gen(struct parser_params*, ID); #define RE_OPTION_MASK 0xff #define RE_OPTION_ARG_ENCODING_NONE 32 -#define NODE_STRTERM NODE_ZARRAY /* nothing to gc */ -#define NODE_HEREDOC NODE_ARRAY /* 1, 3 to gc */ -#define SIGN_EXTEND(x,n) (((1<<(n)-1)^((x)&~(~0<<(n))))-(1<<(n)-1)) -#define nd_func u1.id -#if SIZEOF_SHORT == 2 -#define nd_term(node) ((signed short)(node)->u2.id) -#else -#define nd_term(node) SIGN_EXTEND((node)->u2.id, CHAR_BIT*2) -#endif -#define nd_paren(node) (char)((node)->u2.id >> CHAR_BIT*2) -#define nd_nest u3.cnt +/* structs for managing terminator of string literal and heredocment */ +typedef struct rb_strterm_literal_struct { + long nest; + long func; /* STR_FUNC_* (e.g., STR_FUNC_ESCAPE and STR_FUNC_EXPAND) */ + long paren; /* '(' of `%q(...)` */ + long term; /* ')' of `%q(...)` */ +} rb_strterm_literal_t; + +typedef struct rb_strterm_heredoc_struct { + VALUE sourceline; + VALUE term; /* `"END"` of `<<"END"` */ + long lastidx; /* the column of `<<"END"` */ + VALUE lastline; /* the string of line that contains `<<"END"` */ +} rb_strterm_heredoc_t; + +#define STRTERM_HEREDOC IMEMO_FL_USER0 + +typedef struct rb_strterm_struct { + VALUE flags; + union { + rb_strterm_literal_t literal; + rb_strterm_heredoc_t heredoc; + } u; +} rb_strterm_t; + +void +rb_strterm_mark(VALUE obj) +{ + rb_strterm_t *strterm = (rb_strterm_t*)obj; + if (RBASIC(obj)->flags & STRTERM_HEREDOC) { + rb_strterm_heredoc_t *heredoc = &strterm->u.heredoc; + rb_gc_mark(heredoc->term); + rb_gc_mark(heredoc->lastline); + } +} #define TOKEN2ID(tok) ( \ tTOKEN_LOCAL_BEGIN<(tok)&&(tok)<tTOKEN_LOCAL_END ? TOKEN2LOCALID(tok) : \ @@ -947,6 +973,7 @@ static void token_info_pop_gen(struct parser_params*, const char *token, size_t ID id; int num; const struct vtable *vars; + struct rb_strterm_struct *strterm; } %token <id> @@ -4214,13 +4241,14 @@ string_content : tSTRING_CONTENT } | tSTRING_DVAR { - $<node>$ = lex_strterm; + /* need to backup lex_stream so that a string literal `%&foo,#$&,bar&` can be parsed */ + $<strterm>$ = lex_strterm; lex_strterm = 0; SET_LEX_STATE(EXPR_BEG); } string_dvar { - lex_strterm = $<node>2; + lex_strterm = $<strterm>2; /*%%%*/ $$ = NEW_EVSTR($3); nd_set_lineno($$, @1.first_line); @@ -4237,7 +4265,8 @@ string_content : tSTRING_CONTENT CMDARG_SET(0); } { - $<node>$ = lex_strterm; + /* need to backup lex_stream so that a string literal `%!foo,#{ !0 },bar!` can be parsed */ + $<strterm>$ = lex_strterm; lex_strterm = 0; } { @@ -4256,7 +4285,7 @@ string_content : tSTRING_CONTENT { COND_SET($<val>1); CMDARG_SET($<val>2); - lex_strterm = $<node>3; + lex_strterm = $<strterm>3; SET_LEX_STATE($<num>4); brace_nest = $<num>5; heredoc_indent = $<num>6; @@ -5135,8 +5164,8 @@ none : /* none */ static int parser_regx_options(struct parser_params*); static int parser_tokadd_string(struct parser_params*,int,int,int,long*,rb_encoding**); static void parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc); -static enum yytokentype parser_parse_string(struct parser_params*,NODE*); -static enum yytokentype parser_here_document(struct parser_params*,NODE*); +static enum yytokentype parser_parse_string(struct parser_params*,rb_strterm_literal_t*); +static enum yytokentype parser_here_document(struct parser_params*,rb_strterm_heredoc_t*); # define nextc() parser_nextc(parser) @@ -6466,8 +6495,9 @@ parser_tokadd_string(struct parser_params *parser, return c; } +/* imemo_strterm for literal */ #define NEW_STRTERM(func, term, paren) \ - rb_node_newnode(NODE_STRTERM, (func), (term) | ((paren) << (CHAR_BIT * 2)), 0) + (rb_strterm_t*)rb_imemo_new(imemo_strterm, (VALUE)(func), (VALUE)(paren), (VALUE)(term), 0) #ifdef RIPPER static void @@ -6566,7 +6596,6 @@ parser_peek_variable_name(struct parser_params *parser) static inline enum yytokentype parser_string_term(struct parser_params *parser, int func) { - rb_discard_node(lex_strterm); lex_strterm = 0; if (func & STR_FUNC_REGEXP) { set_yylval_num(regx_options()); @@ -6584,11 +6613,11 @@ parser_string_term(struct parser_params *parser, int func) } static enum yytokentype -parser_parse_string(struct parser_params *parser, NODE *quote) +parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote) { - int func = (int)quote->nd_func; - int term = nd_term(quote); - int paren = nd_paren(quote); + int func = (int)quote->func; + int term = (int)quote->term; + int paren = (int)quote->paren; int c, space = 0; rb_encoding *enc = current_enc; VALUE lit; @@ -6603,9 +6632,9 @@ parser_parse_string(struct parser_params *parser, NODE *quote) do {c = nextc();} while (ISSPACE(c)); space = 1; } - if (c == term && !quote->nd_nest) { + if (c == term && !quote->nest) { if (func & STR_FUNC_QWORDS) { - quote->nd_func |= STR_FUNC_TERM; + quote->func |= STR_FUNC_TERM; return ' '; } return parser_string_term(parser, func); @@ -6622,7 +6651,7 @@ parser_parse_string(struct parser_params *parser, NODE *quote) c = nextc(); } pushback(c); - if (tokadd_string(func, term, paren, "e->nd_nest, + if (tokadd_string(func, term, paren, "e->nest, &enc) == -1) { if (parser->eofp) { #ifndef RIPPER @@ -6637,7 +6666,7 @@ parser_parse_string(struct parser_params *parser, NODE *quote) else { unterminated_literal("unterminated string meets end of file"); } - quote->nd_func |= STR_FUNC_TERM; + quote->func |= STR_FUNC_TERM; } } @@ -6657,7 +6686,6 @@ parser_heredoc_identifier(struct parser_params *parser) long len; int newline = 0; int indent = 0; - VALUE lit; if (c == '-') { c = nextc(); @@ -6722,13 +6750,14 @@ parser_heredoc_identifier(struct parser_params *parser) dispatch_scan_event(tHEREDOC_BEG); len = lex_p - lex_pbeg; lex_goto_eol(parser); - add_mark_object(lit = STR_NEW(tok(), toklen())); - add_mark_object(lex_lastline); - lex_strterm = rb_node_newnode(NODE_HEREDOC, - lit, /* nd_lit */ - len, /* nd_nth */ - lex_lastline); /* nd_orig */ - parser_set_line(lex_strterm, ruby_sourceline); + + lex_strterm = (rb_strterm_t*)rb_imemo_new(imemo_strterm, + STR_NEW(tok(), toklen()), /* term */ + len, /* lastidx */ + lex_lastline, /* lastline */ + ruby_sourceline); + lex_strterm->flags |= STRTERM_HEREDOC; + token_flush(parser); heredoc_indent = indent; heredoc_line_indent = 0; @@ -6736,20 +6765,18 @@ parser_heredoc_identifier(struct parser_params *parser) } static void -parser_heredoc_restore(struct parser_params *parser, NODE *here) +parser_heredoc_restore(struct parser_params *parser, rb_strterm_heredoc_t *here) { VALUE line; lex_strterm = 0; - line = here->nd_orig; + line = here->lastline; lex_lastline = line; lex_pbeg = RSTRING_PTR(line); lex_pend = lex_pbeg + RSTRING_LEN(line); - lex_p = lex_pbeg + here->nd_nth; + lex_p = lex_pbeg + here->lastidx; heredoc_end = ruby_sourceline; - ruby_sourceline = nd_line(here); - dispose_string(parser, here->nd_lit); - rb_discard_node(here); + ruby_sourceline = here->sourceline; token_flush(parser); } @@ -6945,7 +6972,7 @@ ripper_dispatch_heredoc_end(struct parser_params *parser) #endif static enum yytokentype -parser_here_document(struct parser_params *parser, NODE *here) +parser_here_document(struct parser_params *parser, rb_strterm_heredoc_t *here) { int c, func, indent = 0; const char *eos, *p, *pend; @@ -6953,8 +6980,8 @@ parser_here_document(struct parser_params *parser, NODE *here) VALUE str = 0; rb_encoding *enc = current_enc; - eos = RSTRING_PTR(here->nd_lit); - len = RSTRING_LEN(here->nd_lit) - 1; + eos = RSTRING_PTR(here->term); + len = RSTRING_LEN(here->term) - 1; indent = (func = *eos++) & STR_FUNC_INDENT; if ((c = nextc()) == -1) { @@ -6985,13 +7012,13 @@ parser_here_document(struct parser_params *parser, NODE *here) lex_goto_eol(parser); #endif restore: - heredoc_restore(lex_strterm); + heredoc_restore(&lex_strterm->u.heredoc); lex_strterm = 0; return 0; } if (was_bol() && whole_match_p(eos, len, indent)) { dispatch_heredoc_end(); - heredoc_restore(lex_strterm); + heredoc_restore(&lex_strterm->u.heredoc); lex_strterm = 0; SET_LEX_STATE(EXPR_END); return tSTRING_END; @@ -7085,7 +7112,7 @@ parser_here_document(struct parser_params *parser, NODE *here) str = ripper_new_yylval(ripper_token2eventid(tSTRING_CONTENT), yylval.val, str); #endif - heredoc_restore(lex_strterm); + heredoc_restore(&lex_strterm->u.heredoc); lex_strterm = NEW_STRTERM(func | STR_FUNC_TERM, 0, 0); set_yylval_str(str); add_mark_object(str); @@ -8243,11 +8270,11 @@ parser_yylex(struct parser_params *parser) int token_seen = parser->token_seen; if (lex_strterm) { - if (nd_type(lex_strterm) == NODE_HEREDOC) { - return here_document(lex_strterm); + if (lex_strterm->flags & STRTERM_HEREDOC) { + return here_document(&lex_strterm->u.heredoc); } else { - return parse_string(lex_strterm); + return parse_string(&lex_strterm->u.literal); } } cmd_state = command_start; @@ -11542,12 +11569,12 @@ parser_mark(void *ptr) rb_gc_mark(lex_lastline); rb_gc_mark(lex_nextline); rb_gc_mark(ruby_sourcefile_string); + rb_gc_mark((VALUE)lex_strterm); #ifndef RIPPER rb_gc_mark(ruby_debug_lines); rb_gc_mark(parser->compile_option); rb_gc_mark(parser->error_buffer); #else - rb_gc_mark((VALUE)lex_strterm); rb_gc_mark(parser->delayed); rb_gc_mark(parser->value); rb_gc_mark(parser->result); |