Refactor parser compile functions

Refactor parser compile functions to reduce the dependence
on ruby functions.
This commit includes these changes

1. Refactor `gets`, `input` and `gets_` of `parser_params`

Parser needs two different data structure to get next line, function (`gets`) and input data (`input`).
However `gets_` is used for both function (`call`) and input data (`ptr`).
`call` is used for managing general callback function when `rb_ruby_parser_compile_generic` is used.
`ptr` is used for managing the current pointer on String when `parser_compile_string` is used.
This commit changes parser to used only `gets` and `input` then removes `gets_`.

2. Move parser_compile functions and `gets` functions from parse.y to ruby_parser.c

This change reduces the dependence on ruby functions from parser.

3. Change ruby_parser and ripper to take care of `VALUE input` GC mark

Move the responsibility of calling `rb_gc_mark` for `VALUE input` from parser to ruby_parser and ripper.
`input` is arbitrary data pointer from the viewpoint of parser.

4. Introduce rb_parser_compile_array function

Caller of `rb_parser_compile_generic` needs to take care about GC because ruby_parser doesn’t know
about the detail of `lex_gets` and `input`.
Introduce `rb_parser_compile_array` to reduce the complexity of ast.c.

2 files changed, 12 insertions, 6 deletions

diff --git a/internal/parse.h b/internal/parse.h
index e882a16429..73a1c2d1b1 100644
--- a/internal/parse.h
+++ b/internal/parse.h
@@ -53,10 +53,9 @@ void rb_ruby_parser_set_options(rb_parser_t *p, int print, int loop, int chomp,
 rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, int main);
 void rb_ruby_parser_set_script_lines(rb_parser_t *p);
 void rb_ruby_parser_error_tolerant(rb_parser_t *p);
-rb_ast_t* rb_ruby_parser_compile_file_path(rb_parser_t *p, VALUE fname, VALUE file, int start);
 void rb_ruby_parser_keep_tokens(rb_parser_t *p);
-rb_ast_t* rb_ruby_parser_compile_generic(rb_parser_t *p, VALUE (*lex_gets)(VALUE, int), VALUE fname, VALUE input, int start);
-rb_ast_t* rb_ruby_parser_compile_string_path(rb_parser_t *p, VALUE f, VALUE s, int line);
+typedef VALUE (rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int);
+rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line);
 
 RUBY_SYMBOL_EXPORT_BEGIN
 
@@ -91,7 +90,7 @@ VALUE rb_ruby_parser_debug_output(rb_parser_t *p);
 void rb_ruby_parser_set_debug_output(rb_parser_t *p, VALUE output);
 VALUE rb_ruby_parser_parsing_thread(rb_parser_t *p);
 void rb_ruby_parser_set_parsing_thread(rb_parser_t *p, VALUE parsing_thread);
-void rb_ruby_parser_ripper_initialize(rb_parser_t *p, VALUE (*gets)(struct parser_params*,VALUE), VALUE input, VALUE sourcefile_string, const char *sourcefile, int sourceline);
+void rb_ruby_parser_ripper_initialize(rb_parser_t *p, rb_parser_lex_gets_func *gets, rb_parser_input_data input, VALUE sourcefile_string, const char *sourcefile, int sourceline);
 VALUE rb_ruby_parser_result(rb_parser_t *p);
 rb_encoding *rb_ruby_parser_enc(rb_parser_t *p);
 VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p);
@@ -99,7 +98,6 @@ int rb_ruby_parser_ruby_sourceline(rb_parser_t *p);
 int rb_ruby_parser_lex_state(rb_parser_t *p);
 void rb_ruby_ripper_parse0(rb_parser_t *p);
 int rb_ruby_ripper_dedent_string(rb_parser_t *p, VALUE string, int width);
-VALUE rb_ruby_ripper_lex_get_str(rb_parser_t *p, VALUE s);
 int rb_ruby_ripper_initialized_p(rb_parser_t *p);
 void rb_ruby_ripper_parser_initialize(rb_parser_t *p);
 long rb_ruby_ripper_column(rb_parser_t *p);
diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h
index 2559724480..afb8909429 100644
--- a/internal/ruby_parser.h
+++ b/internal/ruby_parser.h
@@ -5,10 +5,16 @@
 #include "internal/bignum.h"
 #include "internal/compilers.h"
 #include "internal/complex.h"
+#include "internal/parse.h"
 #include "internal/rational.h"
 #include "rubyparser.h"
 #include "vm.h"
 
+struct lex_pointer_string {
+    VALUE str;
+    long ptr;
+};
+
 RUBY_SYMBOL_EXPORT_BEGIN
 #ifdef UNIVERSAL_PARSER
 const rb_parser_config_t *rb_ruby_parser_config(void);
@@ -19,6 +25,7 @@ VALUE rb_parser_new(void);
 rb_ast_t *rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line);
 VALUE rb_str_new_parser_string(rb_parser_string_t *str);
 VALUE rb_str_new_mutable_parser_string(rb_parser_string_t *str);
+VALUE rb_parser_lex_get_str(struct lex_pointer_string *ptr_str);
 
 VALUE rb_node_str_string_val(const NODE *);
 VALUE rb_node_sym_string_val(const NODE *);
@@ -48,7 +55,8 @@ void rb_parser_keep_tokens(VALUE vparser);
 
 rb_ast_t *rb_parser_compile_string(VALUE, const char*, VALUE, int);
 rb_ast_t *rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE input, int line);
-rb_ast_t *rb_parser_compile_generic(VALUE vparser, VALUE (*lex_gets)(VALUE, int), VALUE fname, VALUE input, int line);
+rb_ast_t *rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int line);
+rb_ast_t *rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start);
 
 enum lex_state_bits {
     EXPR_BEG_bit,		/* ignore newline, +/- is a sign. */