string.c: integer overflow

* string.c (rb_str_modify_expand): check integer overflow.
  [ruby-core:75592] [Bug #12390]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55054 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 17 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 9ff19d227d..ff4be9648f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+Wed May 18 14:52:38 2016  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* string.c (rb_str_modify_expand): check integer overflow.
+	  [ruby-core:75592] [Bug #12390]
+
 Wed May 18 13:11:44 2016  NARUSE, Yui  <naruse@ruby-lang.org>
 
 	* re.c (match_ary_subseq): get subseq of match array without creating
diff --git a/string.c b/string.c
index 1e4d867f9d..049b088a0d 100644
--- a/string.c
+++ b/string.c
@@ -1914,6 +1914,9 @@ rb_str_modify_expand(VALUE str, long expand)
     else if (expand > 0) {
 	long len = RSTRING_LEN(str);
 	long capa = len + expand;
+	if (expand >= LONG_MAX - len - termlen) {
+	    rb_raise(rb_eArgError, "string size too big");
+	}
 	if (!STR_EMBED_P(str)) {
 	    REALLOC_N(RSTRING(str)->as.heap.ptr, char, capa + termlen);
 	    RSTRING(str)->as.heap.aux.capa = capa;
diff --git a/test/-ext-/string/test_modify_expand.rb b/test/-ext-/string/test_modify_expand.rb
index 5eb7a02b91..d3f5a17037 100644
--- a/test/-ext-/string/test_modify_expand.rb
+++ b/test/-ext-/string/test_modify_expand.rb
@@ -13,4 +13,13 @@ class Test_StringModifyExpand < Test::Unit::TestCase
       s.replace("")
     CMD
   end
+
+  def test_integer_overflow
+    bug12390 = '[ruby-core:75592] [Bug #12390]'
+    s = Bug::String.new
+    long_max = (1 << (8 * RbConfig::SIZEOF['long'] - 1)) - 1
+    assert_raise(ArgumentError, bug12390) {
+      s.modify_expand!(long_max)
+    }
+  end
 end