* io.c (rb_sys_fail_path): move the definition.

  Move above for using it in set_binary_mode_with_seek_cur().

* io.c (set_binary_mode_with_seek_cur): fix improper seek cursor.
  Seeking file cursor with setting binary mode has possibility to
  cause infinite loop. Fixed the bug and refined error handling.
  Introduced at r34043.

  And cleanups as below.
  Remove unnecessary parentheses of `fptr`.
  Use return value of setmode().

* test/ruby/test_io_m17n.rb
  (TestIO_M17N#test_seek_with_setting_binmode): add a test for abobe.
  [ruby-core:41671] [Bug #5714]



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@34132 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 61 insertions, 26 deletions

diff --git a/ChangeLog b/ChangeLog
index 6d85f67e07..8feaeb8080 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+Mon Dec 26 22:01:19 2011  Hiroshi Shirosaki <h.shirosaki@gmail.com>
+
+	* io.c (rb_sys_fail_path): move the definition.
+	  Move above for using it in set_binary_mode_with_seek_cur().
+
+	* io.c (set_binary_mode_with_seek_cur): fix improper seek cursor.
+	  Seeking file cursor with setting binary mode has possibility to
+	  cause infinite loop. Fixed the bug and refined error handling.
+	  Introduced at r34043.
+
+	  And cleanups as below.
+	  Remove unnecessary parentheses of `fptr`.
+	  Use return value of setmode().
+
+	* test/ruby/test_io_m17n.rb
+	  (TestIO_M17N#test_seek_with_setting_binmode): add a test for abobe.
+	  [ruby-core:41671] [Bug #5714]
+
 Mon Dec 26 17:01:14 2011  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* common.mk (LIBRUBY_A): depends on main.o since r33774.
diff --git a/io.c b/io.c
index 34b8a7189f..f6252e2f48 100644
--- a/io.c
+++ b/io.c
@@ -374,6 +374,8 @@ rb_cloexec_fcntl_dupfd(int fd, int minfd)
 #  endif
 #endif
 
+#define rb_sys_fail_path(path) rb_sys_fail(NIL_P(path) ? 0 : RSTRING_PTR(path))
+
 static int io_fflush(rb_io_t *);
 
 #define NEED_NEWLINE_DECORATOR_ON_READ(fptr) ((fptr)->mode & FMODE_TEXTMODE)
@@ -423,49 +425,55 @@ inline static int set_binary_mode_with_seek_cur(rb_io_t *fptr) {
     ssize_t read_size;
     long i;
     long newlines = 0;
+    long extra_max;
     char *p;
 
-    if (!rb_w32_fd_is_text((fptr)->fd)) return O_BINARY;
+    if (!rb_w32_fd_is_text(fptr->fd)) return O_BINARY;
 
-    if ((fptr)->rbuf.len == 0 || (fptr)->mode & FMODE_DUPLEX) {
-	setmode((fptr)->fd, O_BINARY);
-	return O_TEXT;
+    if (fptr->rbuf.len == 0 || fptr->mode & FMODE_DUPLEX) {
+	return setmode(fptr->fd, O_BINARY);
     }
 
     if (io_fflush(fptr) < 0) {
 	rb_sys_fail(0);
     }
     errno = 0;
-    pos = lseek((fptr)->fd, 0, SEEK_CUR);
+    pos = lseek(fptr->fd, 0, SEEK_CUR);
     if (pos < 0 && errno) {
 	if (errno == ESPIPE)
-	    (fptr)->mode |= FMODE_DUPLEX;
-	setmode((fptr)->fd, O_BINARY);
-	return O_TEXT;
+	    fptr->mode |= FMODE_DUPLEX;
+	return setmode(fptr->fd, O_BINARY);
     }
-    /* add extra offset for '\r' */
-    p = (fptr)->rbuf.ptr+(fptr)->rbuf.off;
-    for (i = 0; i < (fptr)->rbuf.len; i++) {
+    /* add extra offset for removed '\r' in rbuf */
+    extra_max = pos - fptr->rbuf.len;
+    p = fptr->rbuf.ptr + fptr->rbuf.off;
+    for (i = 0; i < fptr->rbuf.len; i++) {
 	if (*p == '\n') newlines++;
+	if (extra_max == newlines) break;
 	p++;
     }
     while (newlines >= 0) {
-	r = lseek((fptr)->fd, pos - (fptr)->rbuf.len - newlines, SEEK_SET);
+	r = lseek(fptr->fd, pos - fptr->rbuf.len - newlines, SEEK_SET);
 	if (newlines == 0) break;
-	if (read_size = _read((fptr)->fd, (fptr)->rbuf.ptr, (fptr)->rbuf.len + newlines)) {
-	    if (read_size == (fptr)->rbuf.len) {
-		lseek((fptr)->fd, r, SEEK_SET);
-		break;
-	    }
-	    else {
-		newlines--;
-	    }
+	if (r < 0) {
+	    newlines--;
+	    continue;
+	}
+	read_size = _read(fptr->fd, fptr->rbuf.ptr, fptr->rbuf.len + newlines);
+	if (read_size < 0) {
+	    rb_sys_fail_path(fptr->pathv);
+	}
+	if (read_size == fptr->rbuf.len) {
+	    lseek(fptr->fd, r, SEEK_SET);
+	    break;
+	}
+	else {
+	    newlines--;
 	}
     }
-    (fptr)->rbuf.off = 0;
-    (fptr)->rbuf.len = 0;
-    setmode((fptr)->fd, O_BINARY);
-    return O_TEXT;
+    fptr->rbuf.off = 0;
+    fptr->rbuf.len = 0;
+    return setmode(fptr->fd, O_BINARY);
 }
 #define SET_BINARY_MODE_WITH_SEEK_CUR(fptr) set_binary_mode_with_seek_cur(fptr)
 
@@ -484,8 +492,6 @@ inline static int set_binary_mode_with_seek_cur(rb_io_t *fptr) {
 #define shutdown(a,b)	0
 #endif
 
-#define rb_sys_fail_path(path) rb_sys_fail(NIL_P(path) ? 0 : RSTRING_PTR(path))
-
 #if defined(_WIN32)
 #define is_socket(fd, path)	rb_w32_is_socket(fd)
 #elif !defined(S_ISSOCK)
diff --git a/test/ruby/test_io_m17n.rb b/test/ruby/test_io_m17n.rb
index 9adf7c8bcb..b7460bf63e 100644
--- a/test/ruby/test_io_m17n.rb
+++ b/test/ruby/test_io_m17n.rb
@@ -2314,4 +2314,15 @@ EOT
       end
     }
   end if /mswin|mingw/ =~ RUBY_PLATFORM
+
+  def test_seek_with_setting_binmode
+    with_tmpdir {
+      str = "a\r\nb\r\nc\r\n\r\n\n\n\n\n\n\n\n"
+      generate_file("tmp", str)
+      open("tmp", "r") do |f|
+        assert_equal("a\n", f.gets)      # text
+        assert_equal("b\r\n", f.read(3)) # binary
+      end
+    }
+  end if /mswin|mingw/ =~ RUBY_PLATFORM
 end