openssl: adjust tests for OpenSSL 1.1.0

This fixes `make test-all TESTS=openssl` with OpenSSL master.

* test/openssl/test_x509name.rb: Don't register OID for 'emailAddress'
  and 'serialNumber'. A recent change in OpenSSL made OBJ_create()
  reject an already existing OID. They were needed to run tests with
  OpenSSL 0.9.6 which is now unsupported.
  https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=52832e470f5fe8c222249ae5b539aeb3c74cdb25
  [ruby-core:75225] [Feature #12324]

* test/openssl/test_ssl_session.rb (test_server_session): Duplicate
  SSL::Session before re-adding to the session store. OpenSSL 1.1.0
  starts rejecting SSL_SESSION once removed by SSL_CTX_remove_session().
  https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=7c2d4fee2547650102cd16d23f8125b76112ae75

* test/openssl/test_pkey_ec.rb (setup): Remove X25519 from @keys. X25519
  is new in OpenSSL 1.1.0 but this is for key agreement and not for
  signing.

* test/openssl/test_pair.rb, test/openssl/test_ssl.rb,
  test/openssl/utils.rb: Set security level to 0 when using aNULL cipher
  suites.

* test/openssl/utils.rb: Use 1024 bits DSA key for client certificates.

* test/openssl/test_engine.rb: Run each test in separate process.
  We can no longer cleanup engines explicitly as ENGINE_cleanup() was
  removed.
  https://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=6d4fb1d59e61aacefa25edc4fe5acfe1ac93f743

* ext/openssl/ossl_engine.c (ossl_engine_s_cleanup): Add a note to the
  RDoc for Engine.cleanup.

* ext/openssl/lib/openssl/digest.rb: Don't define constants for DSS,
  DSS1 and SHA(-0) when using with OpenSSL 1.1.0. They are removed.

* test/openssl/test_digest.rb, test/openssl/test_pkey_dsa.rb,
  test/openssl/test_pkey_dsa.rb, test/openssl/test_ssl.rb,
  test/openssl/test_x509cert.rb, test/openssl/test_x509req.rb: Don't
  test unsupported hash functions.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55314 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 66 insertions, 48 deletions

diff --git a/test/openssl/test_engine.rb b/test/openssl/test_engine.rb
index 9a0da34070..77f6e1a967 100644
--- a/test/openssl/test_engine.rb
+++ b/test/openssl/test_engine.rb
@@ -3,75 +3,93 @@ require_relative 'utils'
 
 class OpenSSL::TestEngine < OpenSSL::TestCase
 
-  def teardown
-    super
-    OpenSSL::Engine.cleanup # [ruby-core:40669]
-    assert_equal(0, OpenSSL::Engine.engines.size)
-  end
-
   def test_engines_free # [ruby-dev:44173]
-    OpenSSL::Engine.load("openssl")
-    OpenSSL::Engine.engines
-    OpenSSL::Engine.engines
+    with_openssl <<-'end;'
+      OpenSSL::Engine.load("openssl")
+      OpenSSL::Engine.engines
+      OpenSSL::Engine.engines
+    end;
   end
 
   def test_openssl_engine_builtin
-    engine = OpenSSL::Engine.load("openssl")
-    assert_equal(true, engine)
-    assert_equal(1, OpenSSL::Engine.engines.size)
+    with_openssl <<-'end;'
+      engine = OpenSSL::Engine.load("openssl")
+      assert_equal(true, engine)
+      assert_equal(1, OpenSSL::Engine.engines.size)
+    end;
   end
 
   def test_openssl_engine_by_id_string
-    engine = get_engine
-    assert_not_nil(engine)
-    assert_equal(1, OpenSSL::Engine.engines.size)
+    with_openssl <<-'end;'
+      engine = get_engine
+      assert_not_nil(engine)
+      assert_equal(1, OpenSSL::Engine.engines.size)
+    end;
   end
 
   def test_openssl_engine_id_name_inspect
-    engine = get_engine
-    assert_equal("openssl", engine.id)
-    assert_not_nil(engine.name)
-    assert_not_nil(engine.inspect)
+    with_openssl <<-'end;'
+      engine = get_engine
+      assert_equal("openssl", engine.id)
+      assert_not_nil(engine.name)
+      assert_not_nil(engine.inspect)
+    end;
   end
 
   def test_openssl_engine_digest_sha1
-    engine = get_engine
-    digest = engine.digest("SHA1")
-    assert_not_nil(digest)
-    data = "test"
-    assert_equal(OpenSSL::Digest::SHA1.digest(data), digest.digest(data))
+    with_openssl <<-'end;'
+      engine = get_engine
+      digest = engine.digest("SHA1")
+      assert_not_nil(digest)
+      data = "test"
+      assert_equal(OpenSSL::Digest::SHA1.digest(data), digest.digest(data))
+    end;
   end
 
   def test_openssl_engine_cipher_rc4
-    engine = get_engine
-    algo = "RC4" #AES is not supported by openssl Engine (<=1.0.0e)
-    data = "a" * 1000
-    key = OpenSSL::Random.random_bytes(16)
-    # suppress message from openssl Engine's RC4 cipher [ruby-core:41026]
-    err_back = $stderr.dup
-    $stderr.reopen(IO::NULL)
-    encrypted = crypt_data(data, key, :encrypt) { engine.cipher(algo) }
-    decrypted = crypt_data(encrypted, key, :decrypt) { OpenSSL::Cipher.new(algo) }
-    assert_equal(data, decrypted)
-  ensure
-    if err_back
-      $stderr.reopen(err_back)
-      err_back.close
-    end
+    with_openssl <<-'end;'
+      begin
+        engine = get_engine
+        algo = "RC4" #AES is not supported by openssl Engine (<=1.0.0e)
+        data = "a" * 1000
+        key = OpenSSL::Random.random_bytes(16)
+        # suppress message from openssl Engine's RC4 cipher [ruby-core:41026]
+        err_back = $stderr.dup
+        $stderr.reopen(IO::NULL)
+        encrypted = crypt_data(data, key, :encrypt) { engine.cipher(algo) }
+        decrypted = crypt_data(encrypted, key, :decrypt) { OpenSSL::Cipher.new(algo) }
+        assert_equal(data, decrypted)
+      ensure
+        if err_back
+          $stderr.reopen(err_back)
+          err_back.close
+        end
+      end
+    end;
   end
 
   private
 
-  def get_engine
-    OpenSSL::Engine.by_id("openssl")
+  # this is required because OpenSSL::Engine methods change global state
+  def with_openssl(code)
+    assert_separately(["-ropenssl"], <<~"end;")
+      require #{__FILE__.dump}
+      include OpenSSL::TestEngine::Utils
+      #{code}
+    end;
   end
 
-  def crypt_data(data, key, mode)
-    cipher = yield
-    cipher.send mode
-    cipher.key = key
-    cipher.update(data) + cipher.final
-  end
+  module Utils
+    def get_engine
+      OpenSSL::Engine.by_id("openssl")
+    end
 
-end if defined?(OpenSSL::TestUtils)
+    def crypt_data(data, key, mode)
+      cipher = yield
+      cipher.send mode
+      cipher.key = key
+      cipher.update(data) + cipher.final
+    end
+  end
 
+end if defined?(OpenSSL::TestUtils) && defined?(OpenSSL::Engine)