From 815fd8536b94b12465bbe08868c65119ef63c508 Mon Sep 17 00:00:00 2001 From: akr Date: Mon, 18 Aug 2008 01:40:01 +0000 Subject: * io.c (io_enc_str_converted): new function. (make_readconv): extracted from io_getc. (more_char): ditto. (appendline): use econv via make_readconv and more_char for code conversion. (prepare_getline_args): don't convert record separator. (rb_io_getline_1): don't use rb_io_getline_fast if enc2 is set. (io_getc): use make_readconv and more_char. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@18684 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 11 +++ io.c | 197 ++++++++++++++++++++++++++++++++++------------ test/ruby/test_io_m17n.rb | 12 ++- 3 files changed, 163 insertions(+), 57 deletions(-) diff --git a/ChangeLog b/ChangeLog index 71ea29d03c..7a2ec2a774 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +Mon Aug 18 10:35:25 2008 Tanaka Akira + + * io.c (io_enc_str_converted): new function. + (make_readconv): extracted from io_getc. + (more_char): ditto. + (appendline): use econv via make_readconv and more_char for code + conversion. + (prepare_getline_args): don't convert record separator. + (rb_io_getline_1): don't use rb_io_getline_fast if enc2 is set. + (io_getc): use make_readconv and more_char. + Mon Aug 18 08:27:44 2008 Kazuhiro NISHIYAMA * common.mk: fix error in uncommon.mk. diff --git a/io.c b/io.c index e39e235eef..34b86439d6 100644 --- a/io.c +++ b/io.c @@ -1398,6 +1398,14 @@ io_enc_str(VALUE str, rb_io_t *fptr) return str; } +static VALUE +io_enc_str_converted(VALUE str, rb_io_t *fptr) +{ + OBJ_TAINT(str); + rb_enc_associate(str, io_read_encoding(fptr)); + return str; +} + static VALUE read_all(rb_io_t *fptr, long siz, VALUE str) { @@ -1736,12 +1744,131 @@ rscheck(const char *rsptr, long rslen, VALUE rs) rb_raise(rb_eRuntimeError, "rs modified"); } +static void +make_readconv(rb_io_t *fptr) +{ + if (!fptr->readconv) { + fptr->readconv = rb_econv_open(fptr->enc2->name, fptr->enc->name, 0); + if (!fptr->readconv) + rb_raise(rb_eIOError, "code converter open failed (%s to %s)", fptr->enc->name, fptr->enc2->name); + fptr->crbuf_off = 0; + fptr->crbuf_len = 0; + fptr->crbuf_capa = 1024; + fptr->crbuf = ALLOC_N(char, fptr->crbuf_capa); + } +} + +static int +more_char(rb_io_t *fptr) +{ + const unsigned char *ss, *sp, *se; + unsigned char *ds, *dp, *de; + rb_econv_result_t res; + int putbackable; + int crbuf_len0; + + if (fptr->crbuf_len == fptr->crbuf_capa) + return 0; /* crbuf full */ + if (fptr->crbuf_len == 0) + fptr->crbuf_off = 0; + else if (fptr->crbuf_off + fptr->crbuf_len == fptr->crbuf_capa) { + memmove(fptr->crbuf, fptr->crbuf+fptr->crbuf_off, fptr->crbuf_len); + fptr->crbuf_off = 0; + } + + crbuf_len0 = fptr->crbuf_len; + + while (1) { + ss = sp = (const unsigned char *)fptr->rbuf + fptr->rbuf_off; + se = sp + fptr->rbuf_len; + ds = dp = (unsigned char *)fptr->crbuf + fptr->crbuf_off + fptr->crbuf_len; + de = (unsigned char *)fptr->crbuf + fptr->crbuf_capa; + res = rb_econv_convert(fptr->readconv, &sp, se, &dp, de, ECONV_PARTIAL_INPUT|ECONV_OUTPUT_FOLLOWED_BY_INPUT); + fptr->rbuf_off += sp - ss; + fptr->rbuf_len -= sp - ss; + fptr->crbuf_len += dp - ds; + + putbackable = rb_econv_putbackable(fptr->readconv); + if (putbackable) { + rb_econv_putback(fptr->readconv, (unsigned char *)fptr->rbuf + fptr->rbuf_off - putbackable, putbackable); + fptr->rbuf_off -= putbackable; + fptr->rbuf_len += putbackable; + } + + rb_econv_check_error(fptr->readconv); + + if (crbuf_len0 != fptr->crbuf_len) + return 0; + + if (res == econv_finished) + return -1; + + if (res == econv_source_buffer_empty) { + if (fptr->rbuf_len == 0) { + rb_thread_wait_fd(fptr->fd); + rb_io_check_closed(fptr); + if (io_fillbuf(fptr) == -1) { + ds = dp = (unsigned char *)fptr->crbuf + fptr->crbuf_off + fptr->crbuf_len; + de = (unsigned char *)fptr->crbuf + fptr->crbuf_capa; + res = rb_econv_convert(fptr->readconv, NULL, NULL, &dp, de, 0); + fptr->crbuf_len += dp - ds; + rb_econv_check_error(fptr->readconv); + } + } + } + } +} + static int appendline(rb_io_t *fptr, int delim, VALUE *strp, long *lp) { VALUE str = *strp; long limit = *lp; + if (fptr->enc2) { + make_readconv(fptr); + while (1) { + const char *p, *e; + int searchlen; + if (fptr->crbuf_len) { + p = fptr->crbuf+fptr->crbuf_off; + searchlen = fptr->crbuf_len; + if (0 < limit && limit < searchlen) + searchlen = limit; + e = memchr(p, delim, searchlen); + if (e) { + if (NIL_P(str)) + *strp = str = rb_str_new(p, e-p+1); + else + rb_str_buf_cat(str, p, e-p+1); + fptr->crbuf_off += e-p+1; + fptr->crbuf_len -= e-p+1; + limit -= e-p+1; + *lp = limit; + return delim; + } + + if (NIL_P(str)) + *strp = str = rb_str_new(p, searchlen); + else + rb_str_buf_cat(str, p, searchlen); + fptr->crbuf_off += searchlen; + fptr->crbuf_len -= searchlen; + limit -= searchlen; + + if (limit == 0) { + *lp = limit; + return (unsigned char)RSTRING_PTR(str)[RSTRING_LEN(str)-1]; + } + } + + if (more_char(fptr) == -1) { + *lp = limit; + return EOF; + } + } + } + while (1) { long pending = READ_DATA_PENDING_COUNT(fptr); if (pending > 0) { @@ -1887,15 +2014,6 @@ prepare_getline_args(int argc, VALUE *argv, VALUE *rsp, long *limit, VALUE io) rb_enc_name(enc_rs)); } } - if (fptr->enc2) { - VALUE rs2; - rs2 = rb_funcall(rs, id_encode, 2, - rb_enc_from_encoding(fptr->enc2), - rb_enc_from_encoding(fptr->enc)); - if (!RTEST(rb_str_equal(rs, rs2))) { - rs = rs2; - } - } } *rsp = rs; *limit = NIL_P(lim) ? -1L : NUM2LONG(lim); @@ -1911,9 +2029,6 @@ rb_io_getline_1(VALUE rs, long limit, VALUE io) GetOpenFile(io, fptr); rb_io_check_readable(fptr); - if (rb_enc_dummy_p(io_input_encoding(fptr)) && rs != rb_default_rs) { - rb_raise(rb_eNotImpError, "gets with delimiter against dummy encoding is not currently supported"); - } if (NIL_P(rs)) { str = read_all(fptr, 0, Qnil); if (RSTRING_LEN(str) == 0) return Qnil; @@ -1921,7 +2036,7 @@ rb_io_getline_1(VALUE rs, long limit, VALUE io) else if (limit == 0) { return rb_enc_str_new(0, 0, io_read_encoding(fptr)); } - else if (rs == rb_default_rs && limit < 0 && + else if (rs == rb_default_rs && limit < 0 && !fptr->enc2 && rb_enc_asciicompat(enc = io_read_encoding(fptr))) { return rb_io_getline_fast(fptr, enc); } @@ -1945,7 +2060,10 @@ rb_io_getline_1(VALUE rs, long limit, VALUE io) } newline = (unsigned char)rsptr[rslen - 1]; - enc = io_input_encoding(fptr); + if (fptr->enc2) + enc = fptr->enc; + else + enc = io_input_encoding(fptr); while ((c = appendline(fptr, newline, &str, &limit)) != EOF) { const char *s, *p, *pp; @@ -1981,7 +2099,8 @@ rb_io_getline_1(VALUE rs, long limit, VALUE io) swallow(fptr, '\n'); } } - if (!NIL_P(str)) str = io_enc_str(str, fptr); + if (!NIL_P(str)) + str = io_enc_str_converted(str, fptr); } if (!NIL_P(str)) { @@ -2263,54 +2382,32 @@ io_getc(rb_io_t *fptr, rb_encoding *enc) if (fptr->enc2) { if (!fptr->readconv) { - fptr->readconv = rb_econv_open(fptr->enc2->name, fptr->enc->name, 0); - if (!fptr->readconv) - rb_raise(rb_eIOError, "code converter open failed (%s to %s)", fptr->enc->name, fptr->enc2->name); - fptr->crbuf_off = 0; - fptr->crbuf_len = 0; - fptr->crbuf_capa = 1024; - fptr->crbuf = ALLOC_N(char, fptr->crbuf_capa); + make_readconv(fptr); } while (1) { - const unsigned char *ss, *sp, *se; - unsigned char *ds, *dp, *de; - rb_econv_result_t res; - int putbackable; if (fptr->crbuf_len) { - r = rb_enc_precise_mbclen(fptr->crbuf+fptr->crbuf_off, fptr->crbuf+fptr->crbuf_off+fptr->crbuf_len, fptr->enc); + r = rb_enc_precise_mbclen(fptr->crbuf+fptr->crbuf_off, + fptr->crbuf+fptr->crbuf_off+fptr->crbuf_len, + fptr->enc); if (!MBCLEN_NEEDMORE_P(r)) break; if (fptr->crbuf_len == fptr->crbuf_capa) { rb_raise(rb_eIOError, "too long character"); } } - if (fptr->rbuf_len == 0) { - if (io_fillbuf(fptr) == -1) { - if (fptr->crbuf_len == 0) - return Qnil; - /* return an incomplete character just before EOF */ - return io_shift_crbuf(fptr, fptr->crbuf_len); - } - } - ss = sp = (const unsigned char *)fptr->rbuf + fptr->rbuf_off; - se = sp + fptr->rbuf_len; - ds = dp = (unsigned char *)fptr->crbuf + fptr->crbuf_off + fptr->crbuf_len; - de = (unsigned char *)fptr->crbuf + fptr->crbuf_capa; - res = rb_econv_convert(fptr->readconv, &sp, se, &dp, de, ECONV_PARTIAL_INPUT|ECONV_OUTPUT_FOLLOWED_BY_INPUT); - fptr->rbuf_off += sp - ss; - fptr->rbuf_len -= sp - ss; - fptr->crbuf_len += dp - ds; - putbackable = rb_econv_putbackable(fptr->readconv); - if (putbackable) { - rb_econv_putback(fptr->readconv, (unsigned char *)fptr->rbuf + fptr->rbuf_off - putbackable, putbackable); - fptr->rbuf_off -= putbackable; - fptr->rbuf_len += putbackable; + + if (more_char(fptr) == -1) { + if (fptr->crbuf_len == 0) + return Qnil; + /* return an incomplete character just before EOF */ + return io_shift_crbuf(fptr, fptr->crbuf_len); } - rb_econv_check_error(fptr->readconv); } if (MBCLEN_INVALID_P(r)) { - r = rb_enc_mbclen(fptr->crbuf+fptr->crbuf_off, fptr->crbuf+fptr->crbuf_off+fptr->crbuf_len, fptr->enc); + r = rb_enc_mbclen(fptr->crbuf+fptr->crbuf_off, + fptr->crbuf+fptr->crbuf_off+fptr->crbuf_len, + fptr->enc); return io_shift_crbuf(fptr, r); } return io_shift_crbuf(fptr, MBCLEN_CHARFOUND_LEN(r)); diff --git a/test/ruby/test_io_m17n.rb b/test/ruby/test_io_m17n.rb index 01af6d8b85..e4afd5031b 100644 --- a/test/ruby/test_io_m17n.rb +++ b/test/ruby/test_io_m17n.rb @@ -161,13 +161,11 @@ EOT with_tmpdir { src = "before \e$B\x23\x30\x23\x31\e(B after".force_encoding("iso-2022-jp") generate_file('tmp', src) - assert_raise(NotImplementedError) do - s = open("tmp", "r:iso-2022-jp:euc-jp") {|f| - f.gets("0".force_encoding("euc-jp")) - } - assert_equal(Encoding.find("euc-jp"), s.encoding) - assert_str_equal(src.encode("euc-jp"), s) - end + s = open("tmp", "r:iso-2022-jp:euc-jp") {|f| + f.gets("0".force_encoding("euc-jp")) + } + assert_equal(Encoding.find("euc-jp"), s.encoding) + assert_str_equal(src.encode("euc-jp"), s) } end -- cgit v1.2.3