Merge branch 'maint'

* maint:
  Ruby/OpenSSL 2.0.6
  test/test_engine: check if RC4 is supported
  test/test_engine: suppress stderr
  ossl.c: make legacy locking callbacks reentrant
  ossl.c: use struct CRYPTO_dynlock_value for non-dynamic locks
  ssl: prevent SSLSocket#sysread* from leaking uninitialized data
  test/test_pair: replace sleep with IO.select
  tool/ruby-openssl-docker: update
  test/test_ssl: do not run NPN tests for LibreSSL >= 2.6.1
  test/test_ssl: skip tmp_ecdh_callback test for LibreSSL >= 2.6.1
  test/test_pair: disable compression
  test/test_ssl: suppress warning in test_alpn_protocol_selection_cancel
  ruby.h: unnormalized Fixnum value
  test/test_pair: fix test_write_nonblock{,_no_exceptions}

4 files changed, 77 insertions, 49 deletions

diff --git a/test/test_bn.rb b/test/test_bn.rb
index 77390286..274afba3 100644
--- a/test/test_bn.rb
+++ b/test/test_bn.rb
@@ -270,6 +270,7 @@ class OpenSSL::TestBN < OpenSSL::TestCase
     assert_equal(1, @e1.cmp(-999))
     assert_equal(0, @e1.ucmp(999))
     assert_equal(0, @e1.ucmp(-999))
+    assert_instance_of(String, @e1.hash.to_s)
   end
 end
 
diff --git a/test/test_engine.rb b/test/test_engine.rb
index 4f3973a7..bb1123d5 100644
--- a/test/test_engine.rb
+++ b/test/test_engine.rb
@@ -52,32 +52,28 @@ class OpenSSL::TestEngine < OpenSSL::TestCase
   end
 
   def test_openssl_engine_cipher_rc4
-    with_openssl <<-'end;'
-      begin
-        engine = get_engine
-        algo = "RC4" #AES is not supported by openssl Engine (<=1.0.0e)
-        data = "a" * 1000
-        key = OpenSSL::Random.random_bytes(16)
-        # suppress message from openssl Engine's RC4 cipher [ruby-core:41026]
-        err_back = $stderr.dup
-        $stderr.reopen(IO::NULL)
-        encrypted = crypt_data(data, key, :encrypt) { engine.cipher(algo) }
-        decrypted = crypt_data(encrypted, key, :decrypt) { OpenSSL::Cipher.new(algo) }
-        assert_equal(data, decrypted)
-      ensure
-        if err_back
-          $stderr.reopen(err_back)
-          err_back.close
-        end
-      end
+    begin
+      OpenSSL::Cipher.new("rc4")
+    rescue OpenSSL::Cipher::CipherError
+      pend "RC4 is not supported"
+    end
+
+    with_openssl(<<-'end;', ignore_stderr: true)
+      engine = get_engine
+      algo = "RC4"
+      data = "a" * 1000
+      key = OpenSSL::Random.random_bytes(16)
+      encrypted = crypt_data(data, key, :encrypt) { engine.cipher(algo) }
+      decrypted = crypt_data(encrypted, key, :decrypt) { OpenSSL::Cipher.new(algo) }
+      assert_equal(data, decrypted)
     end;
   end
 
   private
 
   # this is required because OpenSSL::Engine methods change global state
-  def with_openssl(code)
-    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;")
+  def with_openssl(code, **opts)
+    assert_separately([{ "OSSL_MDEBUG" => nil }, "-ropenssl"], <<~"end;", **opts)
       require #{__FILE__.dump}
       include OpenSSL::TestEngine::Utils
       #{code}
diff --git a/test/test_pair.rb b/test/test_pair.rb
index 89cf41a8..55b62321 100644
--- a/test/test_pair.rb
+++ b/test/test_pair.rb
@@ -24,6 +24,7 @@ module OpenSSL::SSLPairM
       sctx.cert = @svr_cert
       sctx.key = @svr_key
       sctx.tmp_dh_callback = proc { OpenSSL::TestUtils::Fixtures.pkey_dh("dh1024") }
+      sctx.options |= OpenSSL::SSL::OP_NO_COMPRESSION
       ssls = OpenSSL::SSL::SSLServer.new(tcps, sctx)
       ns = ssls.accept
       ssls.close
@@ -217,7 +218,7 @@ module OpenSSL::TestPairM
       assert_nothing_raised("[ruby-core:20298]") { ret = s2.read_nonblock(10) }
       assert_equal("def\n", ret)
       s1.close
-      sleep 0.1
+      IO.select([s2])
       assert_raise(EOFError) { s2.read_nonblock(10) }
     }
   end
@@ -233,49 +234,71 @@ module OpenSSL::TestPairM
       assert_nothing_raised("[ruby-core:20298]") { ret = s2.read_nonblock(10, exception: false) }
       assert_equal("def\n", ret)
       s1.close
-      sleep 0.1
+      IO.select([s2])
       assert_equal(nil, s2.read_nonblock(10, exception: false))
     }
   end
 
-  def write_nonblock(socket, meth, str)
-    ret = socket.send(meth, str)
-    ret.is_a?(Symbol) ? 0 : ret
-  end
+  def test_read_with_outbuf
+    ssl_pair { |s1, s2|
+      s1.write("abc\n")
+      buf = ""
+      ret = s2.read(2, buf)
+      assert_same ret, buf
+      assert_equal "ab", ret
+
+      buf = "garbage"
+      ret = s2.read(2, buf)
+      assert_same ret, buf
+      assert_equal "c\n", ret
 
-  def write_nonblock_no_ex(socket, str)
-    ret = socket.write_nonblock str, exception: false
-    ret.is_a?(Symbol) ? 0 : ret
+      buf = "garbage"
+      assert_equal :wait_readable, s2.read_nonblock(100, buf, exception: false)
+      assert_equal "", buf
+
+      s1.close
+      buf = "garbage"
+      assert_equal nil, s2.read(100, buf)
+      assert_equal "", buf
+    }
   end
 
   def test_write_nonblock
     ssl_pair {|s1, s2|
-      n = 0
-      begin
-        n += write_nonblock s1, :write_nonblock, "a" * 100000
-        n += write_nonblock s1, :write_nonblock, "b" * 100000
-        n += write_nonblock s1, :write_nonblock, "c" * 100000
-        n += write_nonblock s1, :write_nonblock, "d" * 100000
-        n += write_nonblock s1, :write_nonblock, "e" * 100000
-        n += write_nonblock s1, :write_nonblock, "f" * 100000
-      rescue IO::WaitWritable
+      assert_equal 3, s1.write_nonblock("foo")
+      assert_equal "foo", s2.read(3)
+
+      data = "x" * 16384
+      written = 0
+      while true
+        begin
+          written += s1.write_nonblock(data)
+        rescue IO::WaitWritable, IO::WaitReadable
+          break
+        end
       end
-      s1.close
-      assert_equal(n, s2.read.length)
+      assert written > 0
+      assert_equal written, s2.read(written).bytesize
     }
   end
 
   def test_write_nonblock_no_exceptions
     ssl_pair {|s1, s2|
-      n = 0
-      n += write_nonblock_no_ex s1, "a" * 100000
-      n += write_nonblock_no_ex s1, "b" * 100000
-      n += write_nonblock_no_ex s1, "c" * 100000
-      n += write_nonblock_no_ex s1, "d" * 100000
-      n += write_nonblock_no_ex s1, "e" * 100000
-      n += write_nonblock_no_ex s1, "f" * 100000
-      s1.close
-      assert_equal(n, s2.read.length)
+      assert_equal 3, s1.write_nonblock("foo", exception: false)
+      assert_equal "foo", s2.read(3)
+
+      data = "x" * 16384
+      written = 0
+      while true
+        case ret = s1.write_nonblock(data, exception: false)
+        when :wait_readable, :wait_writable
+          break
+        else
+          written += ret
+        end
+      end
+      assert written > 0
+      assert_equal written, s2.read(written).bytesize
     }
   end
 
diff --git a/test/test_ssl.rb b/test/test_ssl.rb
index 3f17ab0d..ab6382d7 100644
--- a/test/test_ssl.rb
+++ b/test/test_ssl.rb
@@ -1015,6 +1015,7 @@ if openssl?(1, 0, 2) || libressl?
     ctx1 = OpenSSL::SSL::SSLContext.new
     ctx1.cert = @svr_cert
     ctx1.key = @svr_key
+    ctx1.tmp_dh_callback = proc { Fixtures.pkey_dh("dh1024") }
     ctx1.alpn_select_cb = -> (protocols) { nil }
     ssl1 = OpenSSL::SSL::SSLSocket.new(sock1, ctx1)
 
@@ -1041,6 +1042,7 @@ end
     pend "TLS 1.2 is not supported" unless tls12_supported?
     pend "NPN is not supported" unless \
       OpenSSL::SSL::SSLContext.method_defined?(:npn_select_cb)
+    pend "LibreSSL 2.6 has broken NPN functions" if libressl?(2, 6, 1)
 
     advertised = ["http/1.1", "spdy/2"]
     ctx_proc = proc { |ctx| ctx.npn_protocols = advertised }
@@ -1061,6 +1063,7 @@ end
     pend "TLS 1.2 is not supported" unless tls12_supported?
     pend "NPN is not supported" unless \
       OpenSSL::SSL::SSLContext.method_defined?(:npn_select_cb)
+    pend "LibreSSL 2.6 has broken NPN functions" if libressl?(2, 6, 1)
 
     advertised = Object.new
     def advertised.each
@@ -1085,6 +1088,7 @@ end
     pend "TLS 1.2 is not supported" unless tls12_supported?
     pend "NPN is not supported" unless \
       OpenSSL::SSL::SSLContext.method_defined?(:npn_select_cb)
+    pend "LibreSSL 2.6 has broken NPN functions" if libressl?(2, 6, 1)
 
     ctx_proc = Proc.new { |ctx| ctx.npn_protocols = ["http/1.1"] }
     start_server_version(:TLSv1_2, ctx_proc) { |port|
@@ -1098,6 +1102,7 @@ end
     pend "TLS 1.2 is not supported" unless tls12_supported?
     pend "NPN is not supported" unless \
       OpenSSL::SSL::SSLContext.method_defined?(:npn_select_cb)
+    pend "LibreSSL 2.6 has broken NPN functions" if libressl?(2, 6, 1)
 
     ctx_proc = Proc.new { |ctx| ctx.npn_protocols = ["a" * 256] }
     start_server_version(:TLSv1_2, ctx_proc) { |port|
@@ -1111,6 +1116,7 @@ end
     pend "TLS 1.2 is not supported" unless tls12_supported?
     pend "NPN is not supported" unless \
       OpenSSL::SSL::SSLContext.method_defined?(:npn_select_cb)
+    pend "LibreSSL 2.6 has broken NPN functions" if libressl?(2, 6, 1)
 
     ctx_proc = Proc.new { |ctx| ctx.npn_protocols = ["http/1.1"] }
     start_server_version(:TLSv1_2, ctx_proc) { |port|
@@ -1241,6 +1247,8 @@ end
     pend "EC is disabled" unless defined?(OpenSSL::PKey::EC)
     pend "tmp_ecdh_callback is not supported" unless \
       OpenSSL::SSL::SSLContext.method_defined?(:tmp_ecdh_callback)
+    pend "LibreSSL 2.6 has broken SSL_CTX_set_tmp_ecdh_callback()" \
+      if libressl?(2, 6, 1)
 
     EnvUtil.suppress_warning do # tmp_ecdh_callback is deprecated (2016-05)
       called = false