string.c: fix integer overflow in enc_strlen() and rb_enc_strlen_cr()

* string.c (enc_strlen, rb_enc_strlen_cr): Avoid signed integer
  overflow. The result type of a pointer subtraction may have the same
  size as long. This fixes String#size returning an negative value on
  i686-linux environment:

    str = "\x00" * ((1<<31)-2))
    str.slice!(-3, 3)
    str.force_encoding("UTF-32BE")
    str << 1234
    p str.size

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56247 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 17 insertions, 2 deletions

diff --git a/ChangeLog b/ChangeLog
index 024669496f..d13d137fa9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+Mon Sep 26 11:06:47 2016  Kazuki Yamaguchi  <k@rhe.jp>
+
+	* string.c (enc_strlen, rb_enc_strlen_cr): Avoid signed integer
+	  overflow. The result type of a pointer subtraction may have the same
+	  size as long. This fixes String#size returning an negative value on
+	  i686-linux environment:
+
+	    str = "\x00" * ((1<<31)-2))
+	    str.slice!(-3, 3)
+	    str.force_encoding("UTF-32BE")
+	    str << 1234
+	    p str.size
+
 Sun Sep 25 22:48:06 2016  namusyaka  <namusyaka@gmail.com>
 
 	* lib/erb.rb (ERB::Compiler::TrimScanner#stag): The :stag accessor
diff --git a/string.c b/string.c
index 6988e08ec1..c537bbf4ab 100644
--- a/string.c
+++ b/string.c
@@ -1527,7 +1527,8 @@ enc_strlen(const char *p, const char *e, rb_encoding *enc, int cr)
     const char *q;
 
     if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) {
-        return (e - p + rb_enc_mbminlen(enc) - 1) / rb_enc_mbminlen(enc);
+	long diff = (long)(e - p);
+	return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc));
     }
 #ifdef NONASCII_MASK
     else if (cr == ENC_CODERANGE_VALID && enc == rb_utf8_encoding()) {
@@ -1609,7 +1610,8 @@ rb_enc_strlen_cr(const char *p, const char *e, rb_encoding *enc, int *cr)
 
     *cr = 0;
     if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) {
-	return (e - p + rb_enc_mbminlen(enc) - 1) / rb_enc_mbminlen(enc);
+	long diff = (long)(e - p);
+	return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc));
     }
     else if (rb_enc_asciicompat(enc)) {
 	c = 0;