Backport "Merge branch 'topic/test-memory-leak'" to maint

* topic/test-memory-leak:
  Enable OSSL_MDEBUG on CI builds
  Add OpenSSL.print_mem_leaks
  test: prepare test PKey instances on demand
  test: let OpenSSL::TestCase include OpenSSL::TestUtils
  Don't define main() when built with --enable-debug

(cherry picked from commit 5c586acc387834ab4e09260937dc21064fc59de4)

Note that fix for new test cases that use the old constants removed by
this is squashed in.

1 files changed, 38 insertions, 35 deletions

diff --git a/test/test_pkey_rsa.rb b/test/test_pkey_rsa.rb
index 93760f74..d603b7ac 100644
--- a/test/test_pkey_rsa.rb
+++ b/test/test_pkey_rsa.rb
@@ -5,8 +5,6 @@ require 'base64'
 if defined?(OpenSSL::TestUtils)
 
 class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
-  RSA1024 = OpenSSL::TestUtils::TEST_KEY_RSA1024
-
   def test_padding
     key = OpenSSL::PKey::RSA.new(512, 3)
 
@@ -71,22 +69,23 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
   end
 
   def test_sign_verify
+    rsa1024 = Fixtures.pkey("rsa1024")
     data = "Sign me!"
-    signature = RSA1024.sign("SHA1", data)
-    assert_equal true, RSA1024.verify("SHA1", signature, data)
+    signature = rsa1024.sign("SHA1", data)
+    assert_equal true, rsa1024.verify("SHA1", signature, data)
 
     signature0 = (<<~'end;').unpack("m")[0]
       oLCgbprPvfhM4pjFQiDTFeWI9Sk+Og7Nh9TmIZ/xSxf2CGXQrptlwo7NQ28+
       WA6YQo8jPH4hSuyWIM4Gz4qRYiYRkl5TDMUYob94zm8Si1HxEiS9354tzvqS
       zS8MLW2BtNPuTubMxTItHGTnOzo9sUg0LAHVFt8kHG2NfKAw/gQ=
     end;
-    assert_equal true, RSA1024.verify("SHA256", signature0, data)
+    assert_equal true, rsa1024.verify("SHA256", signature0, data)
     signature1 = signature0.succ
-    assert_equal false, RSA1024.verify("SHA256", signature1, data)
+    assert_equal false, rsa1024.verify("SHA256", signature1, data)
   end
 
   def test_digest_state_irrelevant_sign
-    key = RSA1024
+    key = Fixtures.pkey("rsa1024")
     digest1 = OpenSSL::Digest::SHA1.new
     digest2 = OpenSSL::Digest::SHA1.new
     data = 'Sign me!'
@@ -97,7 +96,7 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
   end
 
   def test_digest_state_irrelevant_verify
-    key = RSA1024
+    key = Fixtures.pkey("rsa1024")
     digest1 = OpenSSL::Digest::SHA1.new
     digest2 = OpenSSL::Digest::SHA1.new
     data = 'Sign me!'
@@ -116,20 +115,21 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
   end
 
   def test_RSAPrivateKey
+    rsa1024 = Fixtures.pkey("rsa1024")
     asn1 = OpenSSL::ASN1::Sequence([
       OpenSSL::ASN1::Integer(0),
-      OpenSSL::ASN1::Integer(RSA1024.n),
-      OpenSSL::ASN1::Integer(RSA1024.e),
-      OpenSSL::ASN1::Integer(RSA1024.d),
-      OpenSSL::ASN1::Integer(RSA1024.p),
-      OpenSSL::ASN1::Integer(RSA1024.q),
-      OpenSSL::ASN1::Integer(RSA1024.dmp1),
-      OpenSSL::ASN1::Integer(RSA1024.dmq1),
-      OpenSSL::ASN1::Integer(RSA1024.iqmp)
+      OpenSSL::ASN1::Integer(rsa1024.n),
+      OpenSSL::ASN1::Integer(rsa1024.e),
+      OpenSSL::ASN1::Integer(rsa1024.d),
+      OpenSSL::ASN1::Integer(rsa1024.p),
+      OpenSSL::ASN1::Integer(rsa1024.q),
+      OpenSSL::ASN1::Integer(rsa1024.dmp1),
+      OpenSSL::ASN1::Integer(rsa1024.dmq1),
+      OpenSSL::ASN1::Integer(rsa1024.iqmp)
     ])
     key = OpenSSL::PKey::RSA.new(asn1.to_der)
     assert_predicate key, :private?
-    assert_same_rsa RSA1024, key
+    assert_same_rsa rsa1024, key
 
     pem = <<~EOF
     -----BEGIN RSA PRIVATE KEY-----
@@ -149,13 +149,14 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     -----END RSA PRIVATE KEY-----
     EOF
     key = OpenSSL::PKey::RSA.new(pem)
-    assert_same_rsa RSA1024, key
+    assert_same_rsa rsa1024, key
 
-    assert_equal asn1.to_der, RSA1024.to_der
-    assert_equal pem, RSA1024.export
+    assert_equal asn1.to_der, rsa1024.to_der
+    assert_equal pem, rsa1024.export
   end
 
   def test_RSAPrivateKey_encrypted
+    rsa1024 = Fixtures.pkey("rsa1024")
     # key = abcdef
     pem = <<~EOF
     -----BEGIN RSA PRIVATE KEY-----
@@ -178,26 +179,27 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     -----END RSA PRIVATE KEY-----
     EOF
     key = OpenSSL::PKey::RSA.new(pem, "abcdef")
-    assert_same_rsa RSA1024, key
+    assert_same_rsa rsa1024, key
     key = OpenSSL::PKey::RSA.new(pem) { "abcdef" }
-    assert_same_rsa RSA1024, key
+    assert_same_rsa rsa1024, key
 
     cipher = OpenSSL::Cipher.new("aes-128-cbc")
-    exported = RSA1024.to_pem(cipher, "abcdef\0\1")
-    assert_same_rsa RSA1024, OpenSSL::PKey::RSA.new(exported, "abcdef\0\1")
+    exported = rsa1024.to_pem(cipher, "abcdef\0\1")
+    assert_same_rsa rsa1024, OpenSSL::PKey::RSA.new(exported, "abcdef\0\1")
     assert_raise(OpenSSL::PKey::RSAError) {
       OpenSSL::PKey::RSA.new(exported, "abcdef")
     }
   end
 
   def test_RSAPublicKey
+    rsa1024 = Fixtures.pkey("rsa1024")
     asn1 = OpenSSL::ASN1::Sequence([
-      OpenSSL::ASN1::Integer(RSA1024.n),
-      OpenSSL::ASN1::Integer(RSA1024.e)
+      OpenSSL::ASN1::Integer(rsa1024.n),
+      OpenSSL::ASN1::Integer(rsa1024.e)
     ])
     key = OpenSSL::PKey::RSA.new(asn1.to_der)
     assert_not_predicate key, :private?
-    assert_same_rsa dup_public(RSA1024), key
+    assert_same_rsa dup_public(rsa1024), key
 
     pem = <<~EOF
     -----BEGIN RSA PUBLIC KEY-----
@@ -207,10 +209,11 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     -----END RSA PUBLIC KEY-----
     EOF
     key = OpenSSL::PKey::RSA.new(pem)
-    assert_same_rsa dup_public(RSA1024), key
+    assert_same_rsa dup_public(rsa1024), key
   end
 
   def test_PUBKEY
+    rsa1024 = Fixtures.pkey("rsa1024")
     asn1 = OpenSSL::ASN1::Sequence([
       OpenSSL::ASN1::Sequence([
         OpenSSL::ASN1::ObjectId("rsaEncryption"),
@@ -218,14 +221,14 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
       ]),
       OpenSSL::ASN1::BitString(
         OpenSSL::ASN1::Sequence([
-          OpenSSL::ASN1::Integer(RSA1024.n),
-          OpenSSL::ASN1::Integer(RSA1024.e)
+          OpenSSL::ASN1::Integer(rsa1024.n),
+          OpenSSL::ASN1::Integer(rsa1024.e)
         ]).to_der
       )
     ])
     key = OpenSSL::PKey::RSA.new(asn1.to_der)
     assert_not_predicate key, :private?
-    assert_same_rsa dup_public(RSA1024), key
+    assert_same_rsa dup_public(rsa1024), key
 
     pem = <<~EOF
     -----BEGIN PUBLIC KEY-----
@@ -236,14 +239,14 @@ class OpenSSL::TestPKeyRSA < OpenSSL::PKeyTestCase
     -----END PUBLIC KEY-----
     EOF
     key = OpenSSL::PKey::RSA.new(pem)
-    assert_same_rsa dup_public(RSA1024), key
+    assert_same_rsa dup_public(rsa1024), key
 
-    assert_equal asn1.to_der, dup_public(RSA1024).to_der
-    assert_equal pem, dup_public(RSA1024).export
+    assert_equal asn1.to_der, dup_public(rsa1024).to_der
+    assert_equal pem, dup_public(rsa1024).export
   end
 
   def test_pem_passwd
-    key = RSA1024
+    key = Fixtures.pkey("rsa1024")
     pem3c = key.to_pem("aes-128-cbc", "key")
     assert_match (/ENCRYPTED/), pem3c
     assert_equal key.to_der, OpenSSL::PKey.read(pem3c, "key").to_der