[ruby/openssl] pkey/dh: deprecate OpenSSL::PKey::DH#generate_key!

OpenSSL::PKey::DH#generate_key! will not work on OpenSSL 3.0 because
keys are made immutable. Users should use OpenSSL::PKey.generate_key
instead.

https://github.com/ruby/openssl/commit/8ee6a582c7

3 files changed, 34 insertions, 16 deletions

diff --git a/ext/openssl/lib/openssl/pkey.rb b/ext/openssl/lib/openssl/pkey.rb
index ba04cf4b39..c3e0629091 100644
--- a/ext/openssl/lib/openssl/pkey.rb
+++ b/ext/openssl/lib/openssl/pkey.rb
@@ -71,14 +71,29 @@ module OpenSSL::PKey
     # called first in order to generate the per-session keys before performing
     # the actual key exchange.
     #
+    # <b>Deprecated in version 3.0</b>. This method is incompatible with
+    # OpenSSL 3.0.0 or later.
+    #
     # See also OpenSSL::PKey.generate_key.
     #
     # Example:
-    #   dh = OpenSSL::PKey::DH.new(2048)
-    #   public_key = dh.public_key #contains no private/public key yet
-    #   public_key.generate_key!
-    #   puts public_key.private? # => true
+    #   # DEPRECATED USAGE: This will not work on OpenSSL 3.0 or later
+    #   dh0 = OpenSSL::PKey::DH.new(2048)
+    #   dh = dh0.public_key # #public_key only copies the DH parameters (contrary to the name)
+    #   dh.generate_key!
+    #   puts dh.private? # => true
+    #   puts dh0.pub_key == dh.pub_key #=> false
+    #
+    #   # With OpenSSL::PKey.generate_key
+    #   dh0 = OpenSSL::PKey::DH.new(2048)
+    #   dh = OpenSSL::PKey.generate_key(dh0)
+    #   puts dh0.pub_key == dh.pub_key #=> false
     def generate_key!
+      if OpenSSL::OPENSSL_VERSION_NUMBER >= 0x30000000
+        raise DHError, "OpenSSL::PKey::DH is immutable on OpenSSL 3.0; " \
+        "use OpenSSL::PKey.generate_key instead"
+      end
+
       unless priv_key
         tmp = OpenSSL::PKey.generate_key(self)
         set_key(tmp.pub_key, tmp.priv_key)
diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index d6f32c62f7..696455dcfd 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -58,15 +58,16 @@ VALUE eDHError;
  *
  * Examples:
  *   # Creating an instance from scratch
- *   dh = DH.new
+ *   # Note that this is deprecated and will not work on OpenSSL 3.0 or later.
+ *   dh = OpenSSL::PKey::DH.new
  *   dh.set_pqg(bn_p, nil, bn_g)
  *
  *   # Generating a parameters and a key pair
- *   dh = DH.new(2048) # An alias of DH.generate(2048)
+ *   dh = OpenSSL::PKey::DH.new(2048) # An alias of OpenSSL::PKey::DH.generate(2048)
  *
  *   # Reading DH parameters
- *   dh = DH.new(File.read('parameters.pem')) # -> dh, but no public/private key yet
- *   dh.generate_key! # -> dh with public and private key
+ *   dh_params = OpenSSL::PKey::DH.new(File.read('parameters.pem')) # loads parameters only
+ *   dh = OpenSSL::PKey.generate_key(dh_params) # generates a key pair
  */
 static VALUE
 ossl_dh_initialize(int argc, VALUE *argv, VALUE self)
diff --git a/test/openssl/test_pkey_dh.rb b/test/openssl/test_pkey_dh.rb
index 757704caf6..ac11af382a 100644
--- a/test/openssl/test_pkey_dh.rb
+++ b/test/openssl/test_pkey_dh.rb
@@ -26,14 +26,19 @@ class OpenSSL::TestPKeyDH < OpenSSL::PKeyTestCase
   end
 
   def test_derive_key
-    dh1 = Fixtures.pkey("dh1024").generate_key!
-    dh2 = Fixtures.pkey("dh1024").generate_key!
+    params = Fixtures.pkey("dh1024")
+    dh1 = OpenSSL::PKey.generate_key(params)
+    dh2 = OpenSSL::PKey.generate_key(params)
     dh1_pub = OpenSSL::PKey.read(dh1.public_to_der)
     dh2_pub = OpenSSL::PKey.read(dh2.public_to_der)
+
     z = dh1.g.mod_exp(dh1.priv_key, dh1.p).mod_exp(dh2.priv_key, dh1.p).to_s(2)
     assert_equal z, dh1.derive(dh2_pub)
     assert_equal z, dh2.derive(dh1_pub)
 
+    assert_raise(OpenSSL::PKey::PKeyError) { params.derive(dh1_pub) }
+    assert_raise(OpenSSL::PKey::PKeyError) { dh1_pub.derive(params) }
+
     assert_equal z, dh1.compute_key(dh2.pub_key)
     assert_equal z, dh2.compute_key(dh1.pub_key)
   end
@@ -74,19 +79,16 @@ class OpenSSL::TestPKeyDH < OpenSSL::PKeyTestCase
   end
 
   def test_generate_key
-    dh = Fixtures.pkey("dh1024").public_key # creates a copy
+    # Deprecated in v3.0.0; incompatible with OpenSSL 3.0
+    dh = Fixtures.pkey("dh1024").public_key # creates a copy with params only
     assert_no_key(dh)
     dh.generate_key!
     assert_key(dh)
-  end
 
-  def test_key_exchange
-    dh = Fixtures.pkey("dh1024")
     dh2 = dh.public_key
-    dh.generate_key!
     dh2.generate_key!
     assert_equal(dh.compute_key(dh2.pub_key), dh2.compute_key(dh.pub_key))
-  end
+  end if !openssl?(3, 0, 0)
 
   def test_params_ok?
     dh0 = Fixtures.pkey("dh1024")