From aa2a845168537c1e495823bfe643924f7f19c28b Mon Sep 17 00:00:00 2001 From: nobu Date: Mon, 29 Jul 2013 08:00:34 +0000 Subject: parse.y, vm_eval.c: file encoding in eval * parse.y (yycompile): store file name as String to keep the encoding. * parse.y (rb_parser_compile_string_path, rb_parser_compile_file_path): new functions to pass file name as a String. * parse.y (gettable_gen): return a copy of the original file name, not a copy in filesystem encoding. * vm_eval.c (eval_string_with_cref): use Qundef instead of "(eval)". git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@42230 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- iseq.c | 28 +++++++++++-------------- node.h | 2 ++ parse.y | 57 ++++++++++++++++++++++++++------------------------ test/ruby/test_eval.rb | 5 +++++ vm_eval.c | 47 +++++++++++++++++++++++------------------ 5 files changed, 76 insertions(+), 63 deletions(-) diff --git a/iseq.c b/iseq.c index 5d8d7ada1f..198295a479 100644 --- a/iseq.c +++ b/iseq.c @@ -574,18 +574,6 @@ rb_iseq_load(VALUE data, VALUE parent, VALUE opt) return iseq_load(rb_cISeq, data, parent, opt); } -static NODE * -parse_string(VALUE str, const char *file, int line) -{ - VALUE parser = rb_parser_new(); - NODE *node = rb_parser_compile_string(parser, file, str, line); - - if (!node) { - rb_exc_raise(GET_THREAD()->errinfo); /* TODO: check err */ - } - return node; -} - VALUE rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE line, rb_block_t *base_block, VALUE opt) { @@ -598,17 +586,25 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE li TH_PUSH_TAG(th); if ((state = EXEC_TAG()) == 0) { + VALUE parser; int ln = NUM2INT(line); - const char *fn = StringValueCStr(file); NODE *node; rb_compile_option_t option; + StringValueCStr(file); make_compile_option(&option, opt); + parser = rb_parser_new(); + if (RB_TYPE_P((src), T_FILE)) - node = rb_compile_file(fn, src, ln); - else - node = parse_string(StringValue(src), fn, ln); + node = rb_parser_compile_file_path(parser, file, src, ln); + else { + node = rb_parser_compile_string_path(parser, file, src, ln); + + if (!node) { + rb_exc_raise(GET_THREAD()->errinfo); /* TODO: check err */ + } + } if (base_block && base_block->iseq) { iseqval = rb_iseq_new_with_opt(node, base_block->iseq->location.label, diff --git a/node.h b/node.h index 57b1bc775a..97c3214d87 100644 --- a/node.h +++ b/node.h @@ -486,6 +486,8 @@ NODE *rb_parser_while_loop(VALUE, NODE *, int, int); NODE *rb_parser_compile_cstr(volatile VALUE, const char*, const char*, int, int); NODE *rb_parser_compile_string(volatile VALUE, const char*, VALUE, int); NODE *rb_parser_compile_file(volatile VALUE, const char*, VALUE, int); +NODE *rb_parser_compile_string_path(volatile VALUE vparser, VALUE fname, VALUE src, int line); +NODE *rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE input, int line); NODE *rb_compile_cstr(const char*, const char*, int, int); NODE *rb_compile_string(const char*, VALUE, int); diff --git a/parse.y b/parse.y index d4e73e0072..7dc4b2e9e4 100644 --- a/parse.y +++ b/parse.y @@ -265,6 +265,7 @@ struct parser_params { int has_shebang; char *parser_ruby_sourcefile; /* current source file */ int parser_ruby_sourceline; /* current line no. */ + VALUE parser_ruby_sourcefile_string; rb_encoding *enc; int parser_yydebug; @@ -281,7 +282,6 @@ struct parser_params { token_info *parser_token_info; #else /* Ripper only */ - VALUE parser_ruby_sourcefile_string; const char *tokp; VALUE delayed; int delayed_line; @@ -338,6 +338,7 @@ static int parser_yyerror(struct parser_params*, const char*); #define ruby__end__seen (parser->parser_ruby__end__seen) #define ruby_sourceline (parser->parser_ruby_sourceline) #define ruby_sourcefile (parser->parser_ruby_sourcefile) +#define ruby_sourcefile_string (parser->parser_ruby_sourcefile_string) #define current_enc (parser->enc) #define yydebug (parser->parser_yydebug) #ifdef RIPPER @@ -5272,14 +5273,13 @@ static void parser_prepare(struct parser_params *parser); #ifndef RIPPER static VALUE -debug_lines(const char *f) +debug_lines(VALUE fname) { ID script_lines; CONST_ID(script_lines, "SCRIPT_LINES__"); if (rb_const_defined_at(rb_cObject, script_lines)) { VALUE hash = rb_const_get_at(rb_cObject, script_lines); if (RB_TYPE_P(hash, T_HASH)) { - VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding()); VALUE lines = rb_ary_new(); rb_hash_aset(hash, fname, lines); return lines; @@ -5289,11 +5289,10 @@ debug_lines(const char *f) } static VALUE -coverage(const char *f, int n) +coverage(VALUE fname, int n) { VALUE coverages = rb_get_coverages(); if (RTEST(coverages) && RBASIC(coverages)->klass == 0) { - VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding()); VALUE lines = rb_ary_new2(n); int i; RBASIC_CLEAR_CLASS(lines); @@ -5319,7 +5318,7 @@ yycompile0(VALUE arg) struct parser_params *parser = (struct parser_params *)arg; if (!compile_for_eval && rb_safe_level() == 0) { - ruby_debug_lines = debug_lines(ruby_sourcefile); + ruby_debug_lines = debug_lines(ruby_sourcefile_string); if (ruby_debug_lines && ruby_sourceline > 0) { VALUE str = STR_NEW0(); n = ruby_sourceline; @@ -5329,7 +5328,7 @@ yycompile0(VALUE arg) } if (!e_option_supplied(parser)) { - ruby_coverage = coverage(ruby_sourcefile, ruby_sourceline); + ruby_coverage = coverage(ruby_sourcefile_string, ruby_sourceline); } } @@ -5372,9 +5371,10 @@ yycompile0(VALUE arg) } static NODE* -yycompile(struct parser_params *parser, const char *f, int line) +yycompile(struct parser_params *parser, VALUE fname, int line) { - ruby_sourcefile = ruby_strdup(f); + ruby_sourcefile_string = rb_str_new_frozen(fname); + ruby_sourcefile = RSTRING_PTR(fname); ruby_sourceline = line - 1; return (NODE *)rb_suppress_tracing(yycompile0, (VALUE)parser); } @@ -5434,7 +5434,7 @@ static rb_data_type_t parser_data_type; static const rb_data_type_t parser_data_type; static NODE* -parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line) +parser_compile_string(volatile VALUE vparser, VALUE fname, VALUE s, int line) { struct parser_params *parser; NODE *node; @@ -5446,7 +5446,7 @@ parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line) lex_pbeg = lex_p = lex_pend = 0; compile_for_eval = rb_parse_in_eval(); - node = yycompile(parser, f, line); + node = yycompile(parser, fname, line); RB_GC_GUARD(vparser); /* prohibit tail call optimization */ return node; @@ -5456,11 +5456,17 @@ NODE* rb_compile_string(const char *f, VALUE s, int line) { must_be_ascii_compatible(s); - return parser_compile_string(rb_parser_new(), f, s, line); + return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), s, line); } NODE* rb_parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line) +{ + return rb_parser_compile_string_path(vparser, rb_filesystem_str_new_cstr(f), s, line); +} + +NODE* +rb_parser_compile_string_path(volatile VALUE vparser, VALUE f, VALUE s, int line) { must_be_ascii_compatible(s); return parser_compile_string(vparser, f, s, line); @@ -5470,14 +5476,14 @@ NODE* rb_compile_cstr(const char *f, const char *s, int len, int line) { VALUE str = rb_str_new(s, len); - return parser_compile_string(rb_parser_new(), f, str, line); + return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), str, line); } NODE* rb_parser_compile_cstr(volatile VALUE vparser, const char *f, const char *s, int len, int line) { VALUE str = rb_str_new(s, len); - return parser_compile_string(vparser, f, str, line); + return parser_compile_string(vparser, rb_filesystem_str_new_cstr(f), str, line); } static VALUE @@ -5496,6 +5502,12 @@ rb_compile_file(const char *f, VALUE file, int start) NODE* rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int start) +{ + return rb_parser_compile_file_path(vparser, rb_filesystem_str_new_cstr(f), file, start); +} + +NODE* +rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE file, int start) { struct parser_params *parser; NODE *node; @@ -5506,7 +5518,7 @@ rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int st lex_pbeg = lex_p = lex_pend = 0; compile_for_eval = rb_parse_in_eval(); - node = yycompile(parser, f, start); + node = yycompile(parser, fname, start); RB_GC_GUARD(vparser); /* prohibit tail call optimization */ return node; @@ -8480,8 +8492,7 @@ gettable_gen(struct parser_params *parser, ID id) case keyword_false: return NEW_FALSE(); case keyword__FILE__: - return NEW_STR(rb_external_str_new_with_enc(ruby_sourcefile, strlen(ruby_sourcefile), - rb_filesystem_encoding())); + return NEW_STR(rb_str_dup(ruby_sourcefile_string)); case keyword__LINE__: return NEW_LIT(INT2FIX(tokline)); case keyword__ENCODING__: @@ -10694,13 +10705,13 @@ parser_initialize(struct parser_params *parser) parser->parser_lvtbl = 0; parser->parser_ruby__end__seen = 0; parser->parser_ruby_sourcefile = 0; + parser->parser_ruby_sourcefile_string = Qnil; #ifndef RIPPER parser->is_ripper = 0; parser->parser_eval_tree_begin = 0; parser->parser_eval_tree = 0; #else parser->is_ripper = 1; - parser->parser_ruby_sourcefile_string = Qnil; parser->delayed = Qnil; parser->result = Qnil; @@ -10728,12 +10739,12 @@ parser_mark(void *ptr) rb_gc_mark(p->parser_lex_input); rb_gc_mark(p->parser_lex_lastline); rb_gc_mark(p->parser_lex_nextline); + rb_gc_mark(p->parser_ruby_sourcefile_string); #ifndef RIPPER rb_gc_mark((VALUE)p->parser_eval_tree_begin) ; rb_gc_mark((VALUE)p->parser_eval_tree) ; rb_gc_mark(p->debug_lines); #else - rb_gc_mark(p->parser_ruby_sourcefile_string); rb_gc_mark(p->delayed); rb_gc_mark(p->value); rb_gc_mark(p->result); @@ -10758,9 +10769,6 @@ parser_free(void *ptr) prev = local->prev; xfree(local); } -#ifndef RIPPER - xfree(p->parser_ruby_sourcefile); -#endif xfree(p); } @@ -10777,11 +10785,6 @@ parser_memsize(const void *ptr) size += sizeof(*local); if (local->vars) size += local->vars->capa * sizeof(ID); } -#ifndef RIPPER - if (p->parser_ruby_sourcefile) { - size += strlen(p->parser_ruby_sourcefile) + 1; - } -#endif return size; } diff --git a/test/ruby/test_eval.rb b/test/ruby/test_eval.rb index 790e2887d5..edbda38818 100644 --- a/test/ruby/test_eval.rb +++ b/test/ruby/test_eval.rb @@ -468,4 +468,9 @@ class TestEval < Test::Unit::TestCase result = foo.instance_eval(&foo_pr) assert_equal(1, result, 'Bug #3786, Bug #3860, [ruby-core:32501]') end + + def test_file_encoding + fname = "\u{3042}".encode("euc-jp") + assert_equal(fname, eval("__FILE__", nil, fname, 1)) + end end diff --git a/vm_eval.c b/vm_eval.c index b5748233f8..ce61849f2c 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1176,7 +1176,7 @@ rb_each(VALUE obj) } static VALUE -eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char *volatile file, volatile int line) +eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, volatile VALUE file, volatile int line) { int state; VALUE result = Qundef; @@ -1188,7 +1188,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char volatile int mild_compile_error; if (file == 0) { - file = rb_sourcefile(); + file = rb_sourcefilename(); line = rb_sourceline(); } @@ -1200,16 +1200,17 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char rb_iseq_t *iseq; volatile VALUE iseqval; VALUE absolute_path = Qnil; + VALUE fname; if (scope != Qnil) { bind = Check_TypedStruct(scope, &ruby_binding_data_type); { envval = bind->env; - if (strcmp(file, "(eval)") != 0) { - absolute_path = rb_str_new_cstr(file); + if (file != Qundef) { + absolute_path = file; } - else if (bind->path != Qnil) { - file = RSTRING_PTR(bind->path); + else if (!NIL_P(bind->path)) { + file = bind->path; line = bind->first_lineno; absolute_path = rb_current_realfilepath(); } @@ -1231,10 +1232,14 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char } } + if ((fname = file) == Qundef) { + fname = rb_usascii_str_new_cstr("(eval)"); + } + /* make eval iseq */ th->parse_in_eval++; th->mild_compile_error++; - iseqval = rb_iseq_compile_with_option(src, rb_str_new2(file), absolute_path, INT2FIX(line), base_block, Qnil); + iseqval = rb_iseq_compile_with_option(src, fname, absolute_path, INT2FIX(line), base_block, Qnil); th->mild_compile_error--; th->parse_in_eval--; @@ -1262,7 +1267,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char if (state) { if (state == TAG_RAISE) { VALUE errinfo = th->errinfo; - if (strcmp(file, "(eval)") == 0) { + if (file == Qundef) { VALUE mesg, errat, bt2; ID id_mesg; @@ -1292,7 +1297,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char } static VALUE -eval_string(VALUE self, VALUE src, VALUE scope, const char *file, int line) +eval_string(VALUE self, VALUE src, VALUE scope, VALUE file, int line) { return eval_string_with_cref(self, src, scope, 0, file, line); } @@ -1319,7 +1324,7 @@ VALUE rb_f_eval(int argc, VALUE *argv, VALUE self) { VALUE src, scope, vfile, vline; - const char *file = "(eval)"; + VALUE file = Qundef; int line = 1; rb_scan_args(argc, argv, "13", &src, &scope, &vfile, &vline); @@ -1341,7 +1346,7 @@ rb_f_eval(int argc, VALUE *argv, VALUE self) } if (!NIL_P(vfile)) - file = RSTRING_PTR(vfile); + file = vfile; return eval_string(self, src, scope, file, line); } @@ -1349,27 +1354,28 @@ rb_f_eval(int argc, VALUE *argv, VALUE self) VALUE ruby_eval_string_from_file(const char *str, const char *filename) { - return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, filename, 1); + VALUE file = filename ? rb_str_new_cstr(filename) : 0; + return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, file, 1); } struct eval_string_from_file_arg { - const char *str; - const char *filename; + VALUE str; + VALUE filename; }; static VALUE eval_string_from_file_helper(VALUE data) { const struct eval_string_from_file_arg *const arg = (struct eval_string_from_file_arg*)data; - return eval_string(rb_vm_top_self(), rb_str_new2(arg->str), Qnil, arg->filename, 1); + return eval_string(rb_vm_top_self(), arg->str, Qnil, arg->filename, 1); } VALUE ruby_eval_string_from_file_protect(const char *str, const char *filename, int *state) { struct eval_string_from_file_arg arg; - arg.str = str; - arg.filename = filename; + arg.str = rb_str_new_cstr(str); + arg.filename = filename ? rb_str_new_cstr(filename) : 0; return rb_protect(eval_string_from_file_helper, (VALUE)&arg, state); } @@ -1529,7 +1535,7 @@ rb_yield_refine_block(VALUE refinement, VALUE refinements) /* string eval under the class/module context */ static VALUE -eval_under(VALUE under, VALUE self, VALUE src, const char *file, int line) +eval_under(VALUE under, VALUE self, VALUE src, VALUE file, int line) { NODE *cref = vm_cref_push(GET_THREAD(), under, NOEX_PUBLIC, NULL); @@ -1554,7 +1560,7 @@ specific_eval(int argc, VALUE *argv, VALUE klass, VALUE self) return yield_under(klass, self, Qundef); } else { - const char *file = "(eval)"; + VALUE file = Qundef; int line = 1; rb_check_arity(argc, 1, 3); @@ -1567,7 +1573,8 @@ specific_eval(int argc, VALUE *argv, VALUE klass, VALUE self) if (argc > 2) line = NUM2INT(argv[2]); if (argc > 1) { - file = StringValuePtr(argv[1]); + file = argv[1]; + if (!NIL_P(file)) StringValue(file); } return eval_under(klass, self, argv[0], file, line); } -- cgit v1.2.3