string.c: fix buffer overflow check condition in rb_str_set_len()

* string.c (rb_str_set_len): The buffer overflow check is wrong. The
  space for termlen is allocated outside the capacity returned by
  rb_str_capacity(). This fixes r41920 ("string.c: multi-byte
  terminator", 2013-07-11).  [ruby-core:77257] [Bug #12757]

* test/-ext-/string/test_set_len.rb (test_capacity_equals_to_new_size):
  Test for this change. Applying only the test will trigger [BUG].

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56148 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 20 insertions, 1 deletions

diff --git a/ChangeLog b/ChangeLog
index 5b1a84a710..920a11cada 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+Tue Sep 13 16:07:26 2016  Kazuki Yamaguchi  <k@rhe.jp>
+
+	* string.c (rb_str_set_len): The buffer overflow check is wrong. The
+	  space for termlen is allocated outside the capacity returned by
+	  rb_str_capacity(). This fixes r41920 ("string.c: multi-byte
+	  terminator", 2013-07-11).  [ruby-core:77257] [Bug #12757]
+
+	* test/-ext-/string/test_set_len.rb (test_capacity_equals_to_new_size):
+	  Test for this change. Applying only the test will trigger [BUG].
+
 Tue Sep 13 06:03:34 2016  NARUSE, Yui  <naruse@ruby-lang.org>
 
 	* common.mk (benchmark): fix lib path.
diff --git a/string.c b/string.c
index 59c44ca4ba..4197a174fb 100644
--- a/string.c
+++ b/string.c
@@ -2497,7 +2497,7 @@ rb_str_set_len(VALUE str, long len)
     if (STR_SHARED_P(str)) {
 	rb_raise(rb_eRuntimeError, "can't set length of shared string");
     }
-    if (len + termlen - 1 > (capa = (long)rb_str_capacity(str))) {
+    if (len > (capa = (long)str_capacity(str, termlen))) {
 	rb_bug("probable buffer overflow: %ld for %ld", len, capa);
     }
     STR_SET_LEN(str, len);
diff --git a/test/-ext-/string/test_set_len.rb b/test/-ext-/string/test_set_len.rb
index 1c5252a5f6..58f51012fb 100644
--- a/test/-ext-/string/test_set_len.rb
+++ b/test/-ext-/string/test_set_len.rb
@@ -23,4 +23,13 @@ class Test_StrSetLen < Test::Unit::TestCase
       assert_equal("abc", @s1.set_len(3))
     }
   end
+
+  def test_capacity_equals_to_new_size
+    bug12757 = "[ruby-core:77257] [Bug #12757]"
+    # fill to ensure capacity does not decrease with force_encoding
+    str = Bug::String.new("\x00" * 128, capacity: 128)
+    str.force_encoding("UTF-32BE")
+    assert_equal 128, Bug::String.capacity(str)
+    assert_equal 127, str.set_len(127).bytesize, bug12757
+  end
 end