openssl: improve handling of password for encrypted PEM

* ext/openssl/ossl.c (ossl_pem_passwd_value): Added. Convert the
  argument to String with StringValue() and validate the length is in
  4..PEM_BUFSIZE. PEM_BUFSIZE is a macro defined in OpenSSL headers.
  (ossl_pem_passwd_cb): When reading/writing encrypted PEM format, we
  used to pass the password to PEM_def_callback() directly but it was
  problematic. It is not NUL character safe. And surprisingly, it
  silently truncates the password to 1024 bytes.  [GH ruby/openssl#51]

* ext/openssl/ossl.h: Add function prototype declaration of newly
  added ossl_pem_passwd_value().

* ext/openssl/ossl_pkey.c (ossl_pkey_new_from_data): Use
  ossl_pem_passwd_value() to validate the password String.

* ext/openssl/ossl_pkey_dsa.c (ossl_dsa_initialize, ossl_dsa_export):
  ditto.

* ext/openssl/ossl_pkey_ec.c (ossl_ec_key_initialize,
  ossl_ec_key_to_string): ditto.

* ext/openssl/ossl_pkey_rsa.c (ossl_rsa_initialize, ossl_rsa_export):
  ditto.

* test/openssl/test_pkey_{dsa,ec,rsa}.rb: test this.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55087 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 9 insertions, 15 deletions

diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index 7e3154afd9..7240de82ab 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -152,27 +152,21 @@ ossl_pkey_new_from_file(VALUE filename)
 static VALUE
 ossl_pkey_new_from_data(int argc, VALUE *argv, VALUE self)
 {
-     EVP_PKEY *pkey;
-     BIO *bio;
-     VALUE data, pass;
-     char *passwd = NULL;
+    EVP_PKEY *pkey;
+    BIO *bio;
+    VALUE data, pass;
 
-     rb_scan_args(argc, argv, "11", &data, &pass);
+    rb_scan_args(argc, argv, "11", &data, &pass);
+    pass = ossl_pem_passwd_value(pass);
 
-     bio = ossl_obj2bio(data);
-     if (!(pkey = d2i_PrivateKey_bio(bio, NULL))) {
+    bio = ossl_obj2bio(data);
+    if (!(pkey = d2i_PrivateKey_bio(bio, NULL))) {
 	OSSL_BIO_reset(bio);
-	if (!NIL_P(pass)) {
-	    passwd = StringValuePtr(pass);
-	}
-	if (!(pkey = PEM_read_bio_PrivateKey(bio, NULL, ossl_pem_passwd_cb, passwd))) {
+	if (!(pkey = PEM_read_bio_PrivateKey(bio, NULL, ossl_pem_passwd_cb, (void *)pass))) {
 	    OSSL_BIO_reset(bio);
 	    if (!(pkey = d2i_PUBKEY_bio(bio, NULL))) {
 		OSSL_BIO_reset(bio);
-		if (!NIL_P(pass)) {
-		    passwd = StringValuePtr(pass);
-		}
-		pkey = PEM_read_bio_PUBKEY(bio, NULL, ossl_pem_passwd_cb, passwd);
+		pkey = PEM_read_bio_PUBKEY(bio, NULL, ossl_pem_passwd_cb, (void *)pass);
 	    }
 	}
     }