openssl: improve handling of password for encrypted PEM

* ext/openssl/ossl.c (ossl_pem_passwd_value): Added. Convert the
  argument to String with StringValue() and validate the length is in
  4..PEM_BUFSIZE. PEM_BUFSIZE is a macro defined in OpenSSL headers.
  (ossl_pem_passwd_cb): When reading/writing encrypted PEM format, we
  used to pass the password to PEM_def_callback() directly but it was
  problematic. It is not NUL character safe. And surprisingly, it
  silently truncates the password to 1024 bytes.  [GH ruby/openssl#51]

* ext/openssl/ossl.h: Add function prototype declaration of newly
  added ossl_pem_passwd_value().

* ext/openssl/ossl_pkey.c (ossl_pkey_new_from_data): Use
  ossl_pem_passwd_value() to validate the password String.

* ext/openssl/ossl_pkey_dsa.c (ossl_dsa_initialize, ossl_dsa_export):
  ditto.

* ext/openssl/ossl_pkey_ec.c (ossl_ec_key_initialize,
  ossl_ec_key_to_string): ditto.

* ext/openssl/ossl_pkey_rsa.c (ossl_rsa_initialize, ossl_rsa_export):
  ditto.

* test/openssl/test_pkey_{dsa,ec,rsa}.rb: test this.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55087 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 42 insertions, 0 deletions

diff --git a/test/openssl/test_pkey_dsa.rb b/test/openssl/test_pkey_dsa.rb
index 2c0e1fc24f..680a123997 100644
--- a/test/openssl/test_pkey_dsa.rb
+++ b/test/openssl/test_pkey_dsa.rb
@@ -218,6 +218,18 @@ YNMbNw==
     assert(pem)
   end
 
+  def test_export_password_funny
+    key = OpenSSL::TestUtils::TEST_KEY_DSA256
+    pem = key.export(OpenSSL::Cipher.new('AES-128-CBC'), "pass\0wd")
+    assert_raise(ArgumentError) do
+      OpenSSL::PKey.read(pem, "pass")
+    end
+    key2 = OpenSSL::PKey.read(pem, "pass\0wd")
+    assert(key2.private?)
+    key3 = OpenSSL::PKey::DSA.new(pem, "pass\0wd")
+    assert(key3.private?)
+  end
+
   private
 
   def check_sign_verify(digest)
diff --git a/test/openssl/test_pkey_ec.rb b/test/openssl/test_pkey_ec.rb
index 673b0d6f43..23afed672a 100644
--- a/test/openssl/test_pkey_ec.rb
+++ b/test/openssl/test_pkey_ec.rb
@@ -184,6 +184,18 @@ class OpenSSL::TestEC < OpenSSL::TestCase
     assert(pem)
   end
 
+  def test_export_password_funny
+    key = OpenSSL::TestUtils::TEST_KEY_EC_P256V1
+    pem = key.export(OpenSSL::Cipher.new('AES-128-CBC'), "pass\0wd")
+    assert_raise(ArgumentError) do
+      OpenSSL::PKey.read(pem, "pass")
+    end
+    key2 = OpenSSL::PKey.read(pem, "pass\0wd")
+    assert(key2.private_key?)
+    key3 = OpenSSL::PKey::EC.new(pem, "pass\0wd")
+    assert(key3.private_key?)
+  end
+
   def test_ec_point_mul
     begin
       # y^2 = x^3 + 2x + 2 over F_17
diff --git a/test/openssl/test_pkey_rsa.rb b/test/openssl/test_pkey_rsa.rb
index 54fce2f59e..c3532f637f 100644
--- a/test/openssl/test_pkey_rsa.rb
+++ b/test/openssl/test_pkey_rsa.rb
@@ -276,6 +276,24 @@ AwEAAQ==
     assert(pem)
   end
 
+  def test_export_password_funny
+    key = OpenSSL::TestUtils::TEST_KEY_RSA1024
+    # this assertion may fail in the future because of OpenSSL change.
+    # the current upper limit is 1024
+    assert_raise(OpenSSL::OpenSSLError) do
+      key.export(OpenSSL::Cipher.new('AES-128-CBC'), 'secr' * 1024)
+    end
+    # password containing NUL byte
+    pem = key.export(OpenSSL::Cipher.new('AES-128-CBC'), "pass\0wd")
+    assert_raise(ArgumentError) do
+      OpenSSL::PKey.read(pem, "pass")
+    end
+    key2 = OpenSSL::PKey.read(pem, "pass\0wd")
+    assert(key2.private?)
+    key3 = OpenSSL::PKey::RSA.new(pem, "pass\0wd")
+    assert(key3.private?)
+  end
+
   private
 
   def check_PUBKEY(asn1, key)