string.c: fix memory leak in rb_str_change_terminator_length()work2

str_make_independent_expand() cannot be used for a String that has its
own buffer.

2 files changed, 24 insertions, 17 deletions

diff --git a/string.c b/string.c
index 5731807379..5bc60fab1e 100644
--- a/string.c
+++ b/string.c
@@ -2161,30 +2161,26 @@ str_fill_term(VALUE str, char *s, long len, int termlen)
 void
 rb_str_change_terminator_length(VALUE str, const int oldtermlen, const int termlen)
 {
-    long capa = str_capacity(str, oldtermlen) + oldtermlen;
+    int diff = termlen - oldtermlen;
     long len = RSTRING_LEN(str);
 
-    assert(capa >= len);
-    if (capa - len < termlen) {
-	rb_check_lockedtmp(str);
-	str_make_independent_expand(str, len, 0L, termlen);
-    }
-    else if (str_dependent_p(str)) {
-	if (termlen > oldtermlen)
+    if (diff > 0) {
+	if (str_dependent_p(str)) {
+	    rb_check_lockedtmp(str);
 	    str_make_independent_expand(str, len, 0L, termlen);
-    }
-    else {
-	if (!STR_EMBED_P(str)) {
-	    /* modify capa instead of realloc */
-	    assert(!FL_TEST((str), STR_SHARED));
-	    RSTRING(str)->as.heap.aux.capa = capa - termlen;
 	}
-	if (termlen > oldtermlen) {
+	else if (str_capacity(str, oldtermlen) - len < (size_t)diff) {
+	    rb_check_lockedtmp(str);
+	    RESIZE_CAPA_TERM(str, len, termlen);
+	}
+	else {
 	    TERM_FILL(RSTRING_PTR(str) + len, termlen);
 	}
     }
-
-    return;
+    else if (diff < 0 &&
+	     FL_TEST(str, STR_NOEMBED|STR_SHARED|STR_NOFREE) == STR_NOEMBED) {
+	RSTRING(str)->as.heap.aux.capa += -diff;
+    }
 }
 
 static char *
diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb
index 7dd76f76f7..c53668ea61 100644
--- a/test/ruby/test_string.rb
+++ b/test/ruby/test_string.rb
@@ -2859,6 +2859,17 @@ CODE
     assert_equal(Encoding::UTF_8, k.encoding, '[ruby-dev:39068]')
   end
 
+  def test_force_encoding_changing_terminator_length
+    assert_no_memory_leak([], 's0 = "x" * 65535', <<~'end;')
+      1_000.times {
+        # Allocate its own buffer of (65535+1) bytes
+        s = s0.b; s.setbyte(0, 0)
+        # Expand it to (65535+4) bytes
+        s.force_encoding("UTF-32LE")
+      }
+    end;
+  end
+
   def test_ascii_incomat_inspect
     bug4081 = '[ruby-core:33283]'
     [Encoding::UTF_16LE, Encoding::UTF_16BE,