pkey: tighten buffer size for signature

We allocate too large buffer for the generated signature. The resulting
signature, or the RSA encryption result, should not be larger than the
size returned by EVP_PKEY_size() (or, DSA_size(), RSA_size(), and
ECDSA_size()).

4 files changed, 9 insertions, 14 deletions

diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index 2ce95b7c..7f22c0df 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -303,7 +303,7 @@ ossl_pkey_sign(VALUE self, VALUE digest, VALUE data)
     pkey = GetPrivPKeyPtr(self);
     md = GetDigestPtr(digest);
     StringValue(data);
-    str = rb_str_new(0, EVP_PKEY_size(pkey)+16);
+    str = rb_str_new(0, EVP_PKEY_size(pkey));
 
     ctx = EVP_MD_CTX_new();
     if (!ctx)
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 3bce66f9..85085419 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -499,8 +499,6 @@ ossl_dsa_to_public_key(VALUE self)
     return obj;
 }
 
-#define ossl_dsa_buf_size(dsa) (DSA_size(dsa) + 16)
-
 /*
  *  call-seq:
  *    dsa.syssign(string) -> aString
@@ -535,7 +533,7 @@ ossl_dsa_sign(VALUE self, VALUE data)
     if (!DSA_PRIVATE(self, dsa))
 	ossl_raise(eDSAError, "Private DSA key needed!");
     StringValue(data);
-    str = rb_str_new(0, ossl_dsa_buf_size(dsa));
+    str = rb_str_new(0, DSA_size(dsa));
     if (!DSA_sign(0, (unsigned char *)RSTRING_PTR(data), RSTRING_LENINT(data),
 		  (unsigned char *)RSTRING_PTR(str),
 		  &buf_len, dsa)) { /* type is ignored (0) */
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index 60f99495..c795c075 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -643,11 +643,10 @@ static VALUE ossl_ec_key_dsa_sign_asn1(VALUE self, VALUE data)
     if (EC_KEY_get0_private_key(ec) == NULL)
 	ossl_raise(eECError, "Private EC key needed!");
 
-    str = rb_str_new(0, ECDSA_size(ec) + 16);
+    str = rb_str_new(0, ECDSA_size(ec));
     if (ECDSA_sign(0, (unsigned char *) RSTRING_PTR(data), RSTRING_LENINT(data), (unsigned char *) RSTRING_PTR(str), &buf_len, ec) != 1)
-         ossl_raise(eECError, "ECDSA_sign");
-
-    rb_str_resize(str, buf_len);
+	ossl_raise(eECError, "ECDSA_sign");
+    rb_str_set_len(str, buf_len);
 
     return str;
 }
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index f969638a..cea228d6 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -404,8 +404,6 @@ ossl_rsa_to_der(VALUE self)
     return str;
 }
 
-#define ossl_rsa_buf_size(rsa) (RSA_size(rsa)+16)
-
 /*
  * call-seq:
  *   rsa.public_encrypt(string)          => String
@@ -429,7 +427,7 @@ ossl_rsa_public_encrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_public_encrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				 (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
@@ -461,7 +459,7 @@ ossl_rsa_public_decrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_public_decrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				 (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
@@ -495,7 +493,7 @@ ossl_rsa_private_encrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_private_encrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				  (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
@@ -529,7 +527,7 @@ ossl_rsa_private_decrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_private_decrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				  (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);