cipher: fix buffer overflow in Cipher#updateky/cipher-update-fix-buffer-size

OpenSSL::Cipher#update currently allocates the output buffer with size
(input data length)+(the block size of the cipher). This is insufficient
for the id-aes{128,192,256}-wrap-pad (AES keywrap with padding) ciphers.
They have a block size of 8 bytes, but the output may be up to 15 bytes
larger than the input.

Use (input data length)+EVP_MAX_BLOCK_LENGTH (== 32) as the output
buffer size, instead. OpenSSL doesn't provide a generic way to tell the
maximum required buffer size for ciphers, but this is large enough for
all algorithms implemented in current versions of OpenSSL.

Fixes: https://bugs.ruby-lang.org/issues/20236

1 files changed, 15 insertions, 3 deletions

diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c
index d9c78914..e5d0f5be 100644
--- a/ext/openssl/ossl_cipher.c
+++ b/ext/openssl/ossl_cipher.c
@@ -387,11 +387,23 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
     if ((in_len = RSTRING_LEN(data)) == 0)
         ossl_raise(rb_eArgError, "data must not be empty");
     GetCipher(self, ctx);
-    out_len = in_len+EVP_CIPHER_CTX_block_size(ctx);
-    if (out_len <= 0) {
+
+    /*
+     * As of OpenSSL 3.2, there is no reliable way to determine the required
+     * output buffer size for arbitrary cipher modes.
+     * https://github.com/openssl/openssl/issues/22628
+     *
+     * in_len+block_size is usually sufficient, but AES key wrap with padding
+     * ciphers require in_len+15 even though they have a block size of 8 bytes.
+     *
+     * Using EVP_MAX_BLOCK_LENGTH (32) as a safe upper bound for ciphers
+     * currently implemented in OpenSSL, but this can change in the future.
+     */
+    if (in_len > LONG_MAX - EVP_MAX_BLOCK_LENGTH) {
 	ossl_raise(rb_eRangeError,
 		   "data too big to make output buffer: %ld bytes", in_len);
     }
+    out_len = in_len + EVP_MAX_BLOCK_LENGTH;
 
     if (NIL_P(str)) {
         str = rb_str_new(0, out_len);
@@ -402,7 +414,7 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
 
     if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len))
 	ossl_raise(eCipherError, NULL);
-    assert(out_len < RSTRING_LEN(str));
+    assert(out_len <= RSTRING_LEN(str));
     rb_str_set_len(str, out_len);
 
     return str;