cipher: fix handling huge data larger than INT_MAX bytes

The function ossl_cipher_update_long() was added to fix this in r48923
(ossl_cipher.c: workaround of OpenSSL API, 2014-12-23), but it didn't
work well.  [Bug #10633]

This can be tested by running:

  $ fallocate -l 2G data.img
  $ ruby -ropenssl <<EOF
  cipher = OpenSSL::Cipher.new("aes-128-ecb").encrypt
  cipher.key = "\x00" * 16
  ct = cipher.update(File.read("data.img")) << cipher.final
  p ct.bytesize
  EOF

1 files changed, 16 insertions, 18 deletions

diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c
index a786ffab..89744a18 100644
--- a/ext/openssl/ossl_cipher.c
+++ b/ext/openssl/ossl_cipher.c
@@ -331,25 +331,23 @@ ossl_cipher_update_long(EVP_CIPHER_CTX *ctx, unsigned char *out, long *out_len_p
 			const unsigned char *in, long in_len)
 {
     int out_part_len;
+    int limit = INT_MAX / 2 + 1;
     long out_len = 0;
-#define UPDATE_LENGTH_LIMIT INT_MAX
-
-#if SIZEOF_LONG > UPDATE_LENGTH_LIMIT
-    if (in_len > UPDATE_LENGTH_LIMIT) {
-	const int in_part_len = (UPDATE_LENGTH_LIMIT / 2 + 1) & ~1;
-	do {
-	    if (!EVP_CipherUpdate(ctx, out ? (out + out_len) : 0,
-				  &out_part_len, in, in_part_len))
-		return 0;
-	    out_len += out_part_len;
-	    in += in_part_len;
-	} while ((in_len -= in_part_len) > UPDATE_LENGTH_LIMIT);
-    }
-#endif
-    if (!EVP_CipherUpdate(ctx, out ? (out + out_len) : 0,
-			  &out_part_len, in, (int)in_len))
-	return 0;
-    if (out_len_ptr) *out_len_ptr = out_len += out_part_len;
+
+    do {
+	int in_part_len = in_len > limit ? limit : in_len;
+
+	if (!EVP_CipherUpdate(ctx, out ? (out + out_len) : 0,
+			      &out_part_len, in, in_part_len))
+	    return 0;
+
+	out_len += out_part_len;
+	in += in_part_len;
+    } while ((in_len -= limit) > 0);
+
+    if (out_len_ptr)
+	*out_len_ptr = out_len;
+
     return 1;
 }