pkey: tighten buffer size for signature

We allocate too large buffer for the generated signature. The resulting
signature, or the RSA encryption result, should not be larger than the
size returned by EVP_PKEY_size() (or, DSA_size(), RSA_size(), and
ECDSA_size()).

4 files changed, 9 insertions, 14 deletions

diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index 2ce95b7..7f22c0d 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -303,7 +303,7 @@ ossl_pkey_sign(VALUE self, VALUE digest, VALUE data)
     pkey = GetPrivPKeyPtr(self);
     md = GetDigestPtr(digest);
     StringValue(data);
-    str = rb_str_new(0, EVP_PKEY_size(pkey)+16);
+    str = rb_str_new(0, EVP_PKEY_size(pkey));
 
     ctx = EVP_MD_CTX_new();
     if (!ctx)
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 3bce66f..8508541 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -499,8 +499,6 @@ ossl_dsa_to_public_key(VALUE self)
     return obj;
 }
 
-#define ossl_dsa_buf_size(dsa) (DSA_size(dsa) + 16)
-
 /*
  *  call-seq:
  *    dsa.syssign(string) -> aString
@@ -535,7 +533,7 @@ ossl_dsa_sign(VALUE self, VALUE data)
     if (!DSA_PRIVATE(self, dsa))
 	ossl_raise(eDSAError, "Private DSA key needed!");
     StringValue(data);
-    str = rb_str_new(0, ossl_dsa_buf_size(dsa));
+    str = rb_str_new(0, DSA_size(dsa));
     if (!DSA_sign(0, (unsigned char *)RSTRING_PTR(data), RSTRING_LENINT(data),
 		  (unsigned char *)RSTRING_PTR(str),
 		  &buf_len, dsa)) { /* type is ignored (0) */
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index 60f9949..c795c07 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -643,11 +643,10 @@ static VALUE ossl_ec_key_dsa_sign_asn1(VALUE self, VALUE data)
     if (EC_KEY_get0_private_key(ec) == NULL)
 	ossl_raise(eECError, "Private EC key needed!");
 
-    str = rb_str_new(0, ECDSA_size(ec) + 16);
+    str = rb_str_new(0, ECDSA_size(ec));
     if (ECDSA_sign(0, (unsigned char *) RSTRING_PTR(data), RSTRING_LENINT(data), (unsigned char *) RSTRING_PTR(str), &buf_len, ec) != 1)
-         ossl_raise(eECError, "ECDSA_sign");
-
-    rb_str_resize(str, buf_len);
+	ossl_raise(eECError, "ECDSA_sign");
+    rb_str_set_len(str, buf_len);
 
     return str;
 }
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index f969638..cea228d 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -404,8 +404,6 @@ ossl_rsa_to_der(VALUE self)
     return str;
 }
 
-#define ossl_rsa_buf_size(rsa) (RSA_size(rsa)+16)
-
 /*
  * call-seq:
  *   rsa.public_encrypt(string)          => String
@@ -429,7 +427,7 @@ ossl_rsa_public_encrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_public_encrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				 (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
@@ -461,7 +459,7 @@ ossl_rsa_public_decrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_public_decrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				 (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
@@ -495,7 +493,7 @@ ossl_rsa_private_encrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_private_encrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				  (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
@@ -529,7 +527,7 @@ ossl_rsa_private_decrypt(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "11", &buffer, &padding);
     pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
     StringValue(buffer);
-    str = rb_str_new(0, ossl_rsa_buf_size(rsa));
+    str = rb_str_new(0, RSA_size(rsa));
     buf_len = RSA_private_decrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
 				  (unsigned char *)RSTRING_PTR(str), rsa, pad);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);