Replace NODE_STRTERM and NODE_HEREDOC with imemo_strterm

Just refactoring.

NODE_STRTERM and NODE_HEREDOC are not an internal node of AST, but a
temporary storage for managing termination of string literals and
heredocs.  Instead of NODE abuse, I want to use imemo for the storage
in order to avoid (my) confusion.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60634 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 86 insertions, 53 deletions

diff --git a/gc.c b/gc.c
index 0d6163ea98..e856fbf9cb 100644
--- a/gc.c
+++ b/gc.c
@@ -4546,6 +4546,9 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj)
       case imemo_ast:
 	rb_ast_mark(&RANY(obj)->as.imemo.ast);
 	return;
+      case imemo_strterm:
+	rb_strterm_mark(obj);
+	return;
 #if VM_CHECK_MODE > 0
       default:
 	VM_UNREACHABLE(gc_mark_imemo);
diff --git a/internal.h b/internal.h
index ad29434c7c..e21a2b6801 100644
--- a/internal.h
+++ b/internal.h
@@ -845,7 +845,8 @@ enum imemo_type {
     imemo_ment       = 6,
     imemo_iseq       = 7,
     imemo_alloc      = 8,
-    imemo_ast        = 9
+    imemo_ast        = 9,
+    imemo_strterm    = 10
 };
 #define IMEMO_MASK   0x0f
 
@@ -941,6 +942,8 @@ typedef struct rb_imemo_alloc_struct {
 
 rb_imemo_alloc_t *rb_imemo_alloc_new(VALUE, VALUE, VALUE, VALUE);
 
+void rb_strterm_mark(VALUE obj);
+
 /*! MEMO
  *
  * @see imemo_type
diff --git a/parse.y b/parse.y
index 79c2bdc18b..364ab691ea 100644
--- a/parse.y
+++ b/parse.y
@@ -178,6 +178,8 @@ typedef struct token_info {
     struct token_info *next;
 } token_info;
 
+typedef struct rb_strterm_struct rb_strterm_t;
+
 /*
     Structure of Lexer Buffer:
 
@@ -193,7 +195,7 @@ struct parser_params {
     YYSTYPE *lval;
 
     struct {
-	NODE *strterm;
+	rb_strterm_t *strterm;
 	VALUE (*gets)(struct parser_params*,VALUE);
 	VALUE input;
 	VALUE lastline;
@@ -738,17 +740,41 @@ static int lvar_defined_gen(struct parser_params*, ID);
 #define RE_OPTION_MASK  0xff
 #define RE_OPTION_ARG_ENCODING_NONE 32
 
-#define NODE_STRTERM NODE_ZARRAY	/* nothing to gc */
-#define NODE_HEREDOC NODE_ARRAY 	/* 1, 3 to gc */
-#define SIGN_EXTEND(x,n) (((1<<(n)-1)^((x)&~(~0<<(n))))-(1<<(n)-1))
-#define nd_func u1.id
-#if SIZEOF_SHORT == 2
-#define nd_term(node) ((signed short)(node)->u2.id)
-#else
-#define nd_term(node) SIGN_EXTEND((node)->u2.id, CHAR_BIT*2)
-#endif
-#define nd_paren(node) (char)((node)->u2.id >> CHAR_BIT*2)
-#define nd_nest u3.cnt
+/* structs for managing terminator of string literal and heredocment */
+typedef struct rb_strterm_literal_struct {
+    long nest;
+    long func;	    /* STR_FUNC_* (e.g., STR_FUNC_ESCAPE and STR_FUNC_EXPAND) */
+    long paren;	    /* '(' of `%q(...)` */
+    long term;	    /* ')' of `%q(...)` */
+} rb_strterm_literal_t;
+
+typedef struct rb_strterm_heredoc_struct {
+    VALUE sourceline;
+    VALUE term;	    /* `"END"` of `<<"END"` */
+    long lastidx;    /* the column of `<<"END"` */
+    VALUE lastline; /* the string of line that contains `<<"END"` */
+} rb_strterm_heredoc_t;
+
+#define STRTERM_HEREDOC IMEMO_FL_USER0
+
+typedef struct rb_strterm_struct {
+    VALUE flags;
+    union {
+	rb_strterm_literal_t literal;
+	rb_strterm_heredoc_t heredoc;
+    } u;
+} rb_strterm_t;
+
+void
+rb_strterm_mark(VALUE obj)
+{
+    rb_strterm_t *strterm = (rb_strterm_t*)obj;
+    if (RBASIC(obj)->flags & STRTERM_HEREDOC) {
+	rb_strterm_heredoc_t *heredoc = &strterm->u.heredoc;
+	rb_gc_mark(heredoc->term);
+	rb_gc_mark(heredoc->lastline);
+    }
+}
 
 #define TOKEN2ID(tok) ( \
     tTOKEN_LOCAL_BEGIN<(tok)&&(tok)<tTOKEN_LOCAL_END ? TOKEN2LOCALID(tok) : \
@@ -947,6 +973,7 @@ static void token_info_pop_gen(struct parser_params*, const char *token, size_t
     ID id;
     int num;
     const struct vtable *vars;
+    struct rb_strterm_struct *strterm;
 }
 
 %token <id>
@@ -4214,13 +4241,14 @@ string_content	: tSTRING_CONTENT
 		    }
 		| tSTRING_DVAR
 		    {
-			$<node>$ = lex_strterm;
+			/* need to backup lex_stream so that a string literal `%&foo,#$&,bar&` can be parsed */
+			$<strterm>$ = lex_strterm;
 			lex_strterm = 0;
 			SET_LEX_STATE(EXPR_BEG);
 		    }
 		  string_dvar
 		    {
-			lex_strterm = $<node>2;
+			lex_strterm = $<strterm>2;
 		    /*%%%*/
 			$$ = NEW_EVSTR($3);
 			nd_set_lineno($$, @1.first_line);
@@ -4237,7 +4265,8 @@ string_content	: tSTRING_CONTENT
 			CMDARG_SET(0);
 		    }
 		    {
-			$<node>$ = lex_strterm;
+			/* need to backup lex_stream so that a string literal `%!foo,#{ !0 },bar!` can be parsed */
+			$<strterm>$ = lex_strterm;
 			lex_strterm = 0;
 		    }
 		    {
@@ -4256,7 +4285,7 @@ string_content	: tSTRING_CONTENT
 		    {
 			COND_SET($<val>1);
 			CMDARG_SET($<val>2);
-			lex_strterm = $<node>3;
+			lex_strterm = $<strterm>3;
 			SET_LEX_STATE($<num>4);
 			brace_nest = $<num>5;
 			heredoc_indent = $<num>6;
@@ -5135,8 +5164,8 @@ none		: /* none */
 static int parser_regx_options(struct parser_params*);
 static int parser_tokadd_string(struct parser_params*,int,int,int,long*,rb_encoding**);
 static void parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc);
-static enum yytokentype parser_parse_string(struct parser_params*,NODE*);
-static enum yytokentype parser_here_document(struct parser_params*,NODE*);
+static enum yytokentype parser_parse_string(struct parser_params*,rb_strterm_literal_t*);
+static enum yytokentype parser_here_document(struct parser_params*,rb_strterm_heredoc_t*);
 
 
 # define nextc()                      parser_nextc(parser)
@@ -6466,8 +6495,9 @@ parser_tokadd_string(struct parser_params *parser,
     return c;
 }
 
+/* imemo_strterm for literal */
 #define NEW_STRTERM(func, term, paren) \
-	rb_node_newnode(NODE_STRTERM, (func), (term) | ((paren) << (CHAR_BIT * 2)), 0)
+	(rb_strterm_t*)rb_imemo_new(imemo_strterm, (VALUE)(func), (VALUE)(paren), (VALUE)(term), 0)
 
 #ifdef RIPPER
 static void
@@ -6566,7 +6596,6 @@ parser_peek_variable_name(struct parser_params *parser)
 static inline enum yytokentype
 parser_string_term(struct parser_params *parser, int func)
 {
-    rb_discard_node(lex_strterm);
     lex_strterm = 0;
     if (func & STR_FUNC_REGEXP) {
 	set_yylval_num(regx_options());
@@ -6584,11 +6613,11 @@ parser_string_term(struct parser_params *parser, int func)
 }
 
 static enum yytokentype
-parser_parse_string(struct parser_params *parser, NODE *quote)
+parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote)
 {
-    int func = (int)quote->nd_func;
-    int term = nd_term(quote);
-    int paren = nd_paren(quote);
+    int func = (int)quote->func;
+    int term = (int)quote->term;
+    int paren = (int)quote->paren;
     int c, space = 0;
     rb_encoding *enc = current_enc;
     VALUE lit;
@@ -6603,9 +6632,9 @@ parser_parse_string(struct parser_params *parser, NODE *quote)
 	do {c = nextc();} while (ISSPACE(c));
 	space = 1;
     }
-    if (c == term && !quote->nd_nest) {
+    if (c == term && !quote->nest) {
 	if (func & STR_FUNC_QWORDS) {
-	    quote->nd_func |= STR_FUNC_TERM;
+	    quote->func |= STR_FUNC_TERM;
 	    return ' ';
 	}
 	return parser_string_term(parser, func);
@@ -6622,7 +6651,7 @@ parser_parse_string(struct parser_params *parser, NODE *quote)
 	c = nextc();
     }
     pushback(c);
-    if (tokadd_string(func, term, paren, &quote->nd_nest,
+    if (tokadd_string(func, term, paren, &quote->nest,
 		      &enc) == -1) {
 	if (parser->eofp) {
 #ifndef RIPPER
@@ -6637,7 +6666,7 @@ parser_parse_string(struct parser_params *parser, NODE *quote)
 	    else {
 		unterminated_literal("unterminated string meets end of file");
 	    }
-	    quote->nd_func |= STR_FUNC_TERM;
+	    quote->func |= STR_FUNC_TERM;
 	}
     }
 
@@ -6657,7 +6686,6 @@ parser_heredoc_identifier(struct parser_params *parser)
     long len;
     int newline = 0;
     int indent = 0;
-    VALUE lit;
 
     if (c == '-') {
 	c = nextc();
@@ -6722,13 +6750,14 @@ parser_heredoc_identifier(struct parser_params *parser)
     dispatch_scan_event(tHEREDOC_BEG);
     len = lex_p - lex_pbeg;
     lex_goto_eol(parser);
-    add_mark_object(lit = STR_NEW(tok(), toklen()));
-    add_mark_object(lex_lastline);
-    lex_strterm = rb_node_newnode(NODE_HEREDOC,
-				  lit,			/* nd_lit */
-				  len,			/* nd_nth */
-				  lex_lastline);	/* nd_orig */
-    parser_set_line(lex_strterm, ruby_sourceline);
+
+    lex_strterm = (rb_strterm_t*)rb_imemo_new(imemo_strterm,
+					      STR_NEW(tok(), toklen()),	/* term */
+					      len,			/* lastidx */
+					      lex_lastline,		/* lastline */
+					      ruby_sourceline);
+    lex_strterm->flags |= STRTERM_HEREDOC;
+
     token_flush(parser);
     heredoc_indent = indent;
     heredoc_line_indent = 0;
@@ -6736,20 +6765,18 @@ parser_heredoc_identifier(struct parser_params *parser)
 }
 
 static void
-parser_heredoc_restore(struct parser_params *parser, NODE *here)
+parser_heredoc_restore(struct parser_params *parser, rb_strterm_heredoc_t *here)
 {
     VALUE line;
 
     lex_strterm = 0;
-    line = here->nd_orig;
+    line = here->lastline;
     lex_lastline = line;
     lex_pbeg = RSTRING_PTR(line);
     lex_pend = lex_pbeg + RSTRING_LEN(line);
-    lex_p = lex_pbeg + here->nd_nth;
+    lex_p = lex_pbeg + here->lastidx;
     heredoc_end = ruby_sourceline;
-    ruby_sourceline = nd_line(here);
-    dispose_string(parser, here->nd_lit);
-    rb_discard_node(here);
+    ruby_sourceline = here->sourceline;
     token_flush(parser);
 }
 
@@ -6945,7 +6972,7 @@ ripper_dispatch_heredoc_end(struct parser_params *parser)
 #endif
 
 static enum yytokentype
-parser_here_document(struct parser_params *parser, NODE *here)
+parser_here_document(struct parser_params *parser, rb_strterm_heredoc_t *here)
 {
     int c, func, indent = 0;
     const char *eos, *p, *pend;
@@ -6953,8 +6980,8 @@ parser_here_document(struct parser_params *parser, NODE *here)
     VALUE str = 0;
     rb_encoding *enc = current_enc;
 
-    eos = RSTRING_PTR(here->nd_lit);
-    len = RSTRING_LEN(here->nd_lit) - 1;
+    eos = RSTRING_PTR(here->term);
+    len = RSTRING_LEN(here->term) - 1;
     indent = (func = *eos++) & STR_FUNC_INDENT;
 
     if ((c = nextc()) == -1) {
@@ -6985,13 +7012,13 @@ parser_here_document(struct parser_params *parser, NODE *here)
 	lex_goto_eol(parser);
 #endif
       restore:
-	heredoc_restore(lex_strterm);
+	heredoc_restore(&lex_strterm->u.heredoc);
 	lex_strterm = 0;
 	return 0;
     }
     if (was_bol() && whole_match_p(eos, len, indent)) {
 	dispatch_heredoc_end();
-	heredoc_restore(lex_strterm);
+	heredoc_restore(&lex_strterm->u.heredoc);
 	lex_strterm = 0;
 	SET_LEX_STATE(EXPR_END);
 	return tSTRING_END;
@@ -7085,7 +7112,7 @@ parser_here_document(struct parser_params *parser, NODE *here)
     str = ripper_new_yylval(ripper_token2eventid(tSTRING_CONTENT),
 			    yylval.val, str);
 #endif
-    heredoc_restore(lex_strterm);
+    heredoc_restore(&lex_strterm->u.heredoc);
     lex_strterm = NEW_STRTERM(func | STR_FUNC_TERM, 0, 0);
     set_yylval_str(str);
     add_mark_object(str);
@@ -8243,11 +8270,11 @@ parser_yylex(struct parser_params *parser)
     int token_seen = parser->token_seen;
 
     if (lex_strterm) {
-	if (nd_type(lex_strterm) == NODE_HEREDOC) {
-	    return here_document(lex_strterm);
+	if (lex_strterm->flags & STRTERM_HEREDOC) {
+	    return here_document(&lex_strterm->u.heredoc);
 	}
 	else {
-	    return parse_string(lex_strterm);
+	    return parse_string(&lex_strterm->u.literal);
 	}
     }
     cmd_state = command_start;
@@ -11542,12 +11569,12 @@ parser_mark(void *ptr)
     rb_gc_mark(lex_lastline);
     rb_gc_mark(lex_nextline);
     rb_gc_mark(ruby_sourcefile_string);
+    rb_gc_mark((VALUE)lex_strterm);
 #ifndef RIPPER
     rb_gc_mark(ruby_debug_lines);
     rb_gc_mark(parser->compile_option);
     rb_gc_mark(parser->error_buffer);
 #else
-    rb_gc_mark((VALUE)lex_strterm);
     rb_gc_mark(parser->delayed);
     rb_gc_mark(parser->value);
     rb_gc_mark(parser->result);