diff options
author | aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2004-09-20 07:59:30 +0000 |
---|---|---|
committer | aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2004-09-20 07:59:30 +0000 |
commit | b2a700f13eb406145983ff2f2308ce0f6b1ebbe5 (patch) | |
tree | c22c02a0c7bf687e952da4055502f0b6c34870ff | |
parent | 273d0344a8c25e2f14129d78df4cc7242762f507 (diff) | |
download | ruby-b2a700f13eb406145983ff2f2308ce0f6b1ebbe5.tar.gz |
* parse.y [ripper]: do not delay dispatching.
* ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6936 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r-- | ChangeLog | 6 | ||||
-rw-r--r-- | ext/ripper/lib/ripper/tokenizer.rb | 6 | ||||
-rw-r--r-- | parse.y | 88 |
3 files changed, 44 insertions, 56 deletions
@@ -1,3 +1,9 @@ +Mon Sep 20 16:58:16 2004 Minero Aoki <aamine@loveruby.net> + + * parse.y [ripper]: do not delay dispatching. + + * ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order. + Mon Sep 20 15:17:47 2004 Minero Aoki <aamine@loveruby.net> * ext/ripper/lib/ripper/tokenizer.rb: new file. diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb index 0bd966ea51..134de9d1e0 100644 --- a/ext/ripper/lib/ripper/tokenizer.rb +++ b/ext/ripper/lib/ripper/tokenizer.rb @@ -24,11 +24,13 @@ class Ripper def tokenize @tokens = [] parse - @tokens + @tokens.sort_by {|tok, pos| pos }.map {|tok,| tok } end + private + def on__scan(type, tok) - @tokens.push tok + @tokens.push [tok, [lineno(),column()]] end end @@ -145,9 +145,9 @@ struct parser_params { int parser_ruby_sourceline; VALUE parser_ruby_sourcefile; char *tokp; - int current_t; - int in_heredoc; VALUE delayed; + int delayed_line; + int delayed_col; #endif }; @@ -4206,46 +4206,26 @@ ripper_dispatch_scan_event(parser, t) if (lex_p == parser->tokp) return; str = rb_str_new(parser->tokp, lex_p - parser->tokp); event = ripper_token2eventid(t); - if (parser->in_heredoc) { - int col = parser->tokp - lex_pbeg; - rb_ary_push(parser->delayed, ID2SYM(event)); - rb_ary_push(parser->delayed, str); - rb_ary_push(parser->delayed, INT2NUM(ruby_sourceline)); - rb_ary_push(parser->delayed, INT2NUM(col)); - ripper_flush(parser); - return; - } ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str)); - ripper_dispatch1(parser, event, str); + yylval.val = ripper_dispatch1(parser, event, str); ripper_flush(parser); } static void -ripper_dispatch_delayed_events(parser) +ripper_dispatch_delayed_token(parser, t) struct parser_params *parser; + int t; { - long i; - int saved_line; - char *saved_tokp; - - if (RARRAY(parser->delayed)->len == 0) return; - if (RARRAY(parser->delayed)->len % 4 != 0) - rb_raise(rb_eRuntimeError, "[RIPPER BUG] parser->delayed % 4 != 0"); - saved_line = ruby_sourceline; - saved_tokp = parser->tokp; - i = 0; - while (i < RARRAY(parser->delayed)->len) { - VALUE event, str, vline, vcol; - event = RARRAY(parser->delayed)->ptr[i++]; - str = RARRAY(parser->delayed)->ptr[i++]; - vline = RARRAY(parser->delayed)->ptr[i++]; - vcol = RARRAY(parser->delayed)->ptr[i++]; - ruby_sourceline = NUM2INT(vline); - parser->tokp = lex_pbeg + NUM2INT(vcol); - ripper_dispatch2(parser, ripper_id_scan, event, rb_str_dup(str)); - ripper_dispatch1(parser, SYM2ID(event), str); - } - rb_ary_clear(parser->delayed); + int saved_line = ruby_sourceline; + char *saved_tokp = parser->tokp; + ID event = ripper_token2eventid(t); + VALUE str = parser->delayed; + + ruby_sourceline = parser->delayed_line; + parser->tokp = lex_pbeg + parser->delayed_col; + ripper_dispatch2(parser, ripper_id_scan, ID2SYM(event), rb_str_dup(str)); + yylval.val = ripper_dispatch1(parser, event, str); + parser->delayed = Qnil; ruby_sourceline = saved_line; parser->tokp = saved_tokp; } @@ -4486,9 +4466,17 @@ parser_nextc(parser) } #ifdef RIPPER if (parser->tokp < lex_pend) { - if (!parser->current_t) - rb_raise(rb_eRuntimeError, "[Ripper BUG] no current_t"); - ripper_dispatch_scan_event(parser, parser->current_t); + if (NIL_P(parser->delayed)) { + parser->delayed = rb_str_buf_new(1024); + rb_str_buf_cat(parser->delayed, + parser->tokp, lex_pend - parser->tokp); + parser->delayed_line = ruby_sourceline; + parser->delayed_col = parser->tokp - lex_pbeg; + } + else { + rb_str_buf_cat(parser->delayed, + parser->tokp, lex_pend - parser->tokp); + } } #endif if (heredoc_end > 0) { @@ -4939,9 +4927,6 @@ parser_parse_string(parser, quote) if (func == -1) return tSTRING_END; c = nextc(); if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) { -#ifdef RIPPER - parser->current_t = ' '; -#endif do {c = nextc();} while (ISSPACE(c)); space = 1; } @@ -5057,6 +5042,8 @@ parser_heredoc_restore(parser, here) VALUE line; #ifdef RIPPER + if (!NIL_P(parser->delayed)) + ripper_dispatch_delayed_token(parser, tSTRING_CONTENT); lex_goto_eol(parser); ripper_dispatch_scan_event(parser, tHEREDOC_END); #endif @@ -5071,7 +5058,6 @@ parser_heredoc_restore(parser, here) rb_gc_force_recycle((VALUE)here); #ifdef RIPPER ripper_flush(parser); - parser->in_heredoc = Qfalse; #endif } @@ -5222,13 +5208,7 @@ parser_yylex(parser) if (lex_strterm) { int token; -#ifdef RIPPER - parser->current_t = tSTRING_CONTENT; -#endif if (nd_type(lex_strterm) == NODE_HEREDOC) { -#ifdef RIPPER - parser->in_heredoc = Qtrue; -#endif token = here_document(lex_strterm); if (token == tSTRING_END) { lex_strterm = 0; @@ -6444,11 +6424,12 @@ yylex(p) #endif t = parser_yylex(parser); #ifdef RIPPER - parser->current_t = 0; + if (!NIL_P(parser->delayed)) { + ripper_dispatch_delayed_token(parser, t); + return t; + } if (t != 0) ripper_dispatch_scan_event(parser, t); - if ((t == '\n' && !parser->in_heredoc) || (t == 0)) - ripper_dispatch_delayed_events(parser); #endif return t; @@ -8137,9 +8118,8 @@ parser_initialize(parser) parser->parser_lex_p = 0; parser->parser_lex_pend = 0; #ifdef RIPPER - parser->current_t = 0; - parser->in_heredoc = Qfalse; - parser->delayed = rb_ary_new(); + parser->parser_ruby_sourcefile = Qnil; + parser->delayed = Qnil; #endif } |