diff options
| -rw-r--r-- | ChangeLog | 6 | ||||
| -rw-r--r-- | parse.y | 30 | ||||
| -rw-r--r-- | version.h | 6 |
3 files changed, 32 insertions, 10 deletions
@@ -1,3 +1,9 @@ +Wed Oct 24 17:09:39 2007 Nobuyoshi Nakada <nobu@ruby-lang.org> + + * parse.y (parser_tokspace): make space in token buffer. + + * parse.y (parser_yylex): fix encoding of single character literal. + Tue Oct 23 13:44:14 2007 Nobuyoshi Nakada <nobu@ruby-lang.org> * parse.y (call_args2): nd_head of NODE_BLOCK_PASS should be a list. @@ -4494,6 +4494,7 @@ static int parser_here_document(struct parser_params*,NODE*); # define nextc() parser_nextc(parser) # define pushback(c) parser_pushback(parser, c) # define newtok() parser_newtok(parser) +# define tokspace(n) parser_tokspace(parser, n) # define tokadd(c) parser_tokadd(parser, c) # define read_escape(m) parser_read_escape(parser, m) # define tokadd_escape(t,m) parser_tokadd_escape(parser, t, m) @@ -4913,6 +4914,18 @@ parser_newtok(struct parser_params *parser) return tokenbuf; } +static char * +parser_tokspace(struct parser_params *parser, int n) +{ + int idx = tokidx + n; + + if (idx >= toksiz) { + do {toksiz *= 2;} while (toksiz < idx); + REALLOC_N(tokenbuf, char, toksiz); + } + return &tokenbuf[tokidx]; +} + static void parser_tokadd(struct parser_params *parser, int c) { @@ -6086,21 +6099,24 @@ parser_yylex(struct parser_params *parser) } newtok(); if (parser_ismbchar()) { + mb = ENC_CODERANGE_MULTI; tokadd_mbchar(c); } else if ((rb_enc_isalnum(c, parser->enc) || c == '_') && lex_p < lex_pend && is_identchar(lex_p, lex_pend, parser->enc)) { goto ternary; } - else if (c == '\\') { - c = read_escape(0); - tokadd(c); + else if (c == '\\' && (c = read_escape(0)) >= 0x80) { + rb_encoding *enc = parser->enc; + mb = ENC_CODERANGE_UNKNOWN; + rb_enc_mbcput(c, tokspace(rb_enc_codelen(c, enc)), enc); } else { + mb = ENC_CODERANGE_SINGLE; tokadd(c); } tokfix(); - set_yylval_str(STR_NEW(tok(), toklen())); + set_yylval_str(STR_NEW3(tok(), toklen(), mb)); lex_state = EXPR_ENDARG; return tCHAR; @@ -6865,9 +6881,9 @@ parser_yylex(struct parser_params *parser) break; } - mb = 0; + mb = ENC_CODERANGE_SINGLE; do { - if (!ISASCII(c)) mb = 1; + if (!ISASCII(c)) mb = ENC_CODERANGE_UNKNOWN; tokadd_mbchar(c); c = nextc(); } while (parser_is_identchar()); @@ -6920,7 +6936,7 @@ parser_yylex(struct parser_params *parser) } } - if (!mb && lex_state != EXPR_DOT) { + if (mb == ENC_CODERANGE_SINGLE && lex_state != EXPR_DOT) { const struct kwtable *kw; /* See if it is a reserved word. */ @@ -1,7 +1,7 @@ #define RUBY_VERSION "1.9.0" -#define RUBY_RELEASE_DATE "2007-10-23" +#define RUBY_RELEASE_DATE "2007-10-24" #define RUBY_VERSION_CODE 190 -#define RUBY_RELEASE_CODE 20071023 +#define RUBY_RELEASE_CODE 20071024 #define RUBY_PATCHLEVEL 0 #define RUBY_VERSION_MAJOR 1 @@ -9,7 +9,7 @@ #define RUBY_VERSION_TEENY 0 #define RUBY_RELEASE_YEAR 2007 #define RUBY_RELEASE_MONTH 10 -#define RUBY_RELEASE_DAY 23 +#define RUBY_RELEASE_DAY 24 #ifdef RUBY_EXTERN RUBY_EXTERN const char ruby_version[]; |