test: adjust test cases for LibreSSL 3.2.4

This is a backport to the 2.1 branch of the following commits:

 - a0e98d48c91f ("Enhance TLS 1.3 support on LibreSSL 3.2/3.3", 2020-12-03)
 - a9954bac22ba ("test: adjust test cases for LibreSSL 3.2.4", 2021-02-25)

4 files changed, 51 insertions, 11 deletions

diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index f4271369..a8386f72 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -13,6 +13,12 @@
 
 #define numberof(ary) (int)(sizeof(ary)/sizeof((ary)[0]))
 
+#if !defined(TLS1_3_VERSION) && \
+    defined(LIBRESSL_VERSION_NUMBER) && \
+    LIBRESSL_VERSION_NUMBER >= 0x3020000fL
+#  define TLS1_3_VERSION 0x0304
+#endif
+
 #ifdef _WIN32
 #  define TO_SOCKET(s) _get_osfhandle(s)
 #else
diff --git a/test/test_ssl.rb b/test/test_ssl.rb
index 764416be..53457e21 100644
--- a/test/test_ssl.rb
+++ b/test/test_ssl.rb
@@ -207,7 +207,10 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
 
   def test_client_auth_success
     vflag = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
-    start_server(verify_mode: vflag) { |port|
+    start_server(verify_mode: vflag,
+      ctx_proc: proc { |ctx|
+        ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
+    }) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.key = @cli_key
       ctx.cert = @cli_cert
@@ -253,6 +256,8 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
   end
 
   def test_client_ca
+    pend "LibreSSL 3.2 has broken client CA support" if libressl?(3, 2, 0)
+
     ctx_proc = Proc.new do |ctx|
       ctx.client_ca = [@ca_cert]
     end
@@ -793,11 +798,13 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
 
   def test_verify_hostname_on_connect
     ctx_proc = proc { |ctx|
+      san = "DNS:a.example.com,DNS:*.b.example.com"
+      san += ",DNS:c*.example.com,DNS:d.*.example.com" unless libressl?(3, 2, 2)
       exts = [
         ["keyUsage", "keyEncipherment,digitalSignature", true],
-        ["subjectAltName", "DNS:a.example.com,DNS:*.b.example.com," \
-                           "DNS:c*.example.com,DNS:d.*.example.com"],
+        ["subjectAltName", san],
       ]
+
       ctx.cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key)
       ctx.key = @svr_key
     }
@@ -818,6 +825,7 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
         ["cx.example.com", true],
         ["d.x.example.com", false],
       ].each do |name, expected_ok|
+        next if name.start_with?('cx') if libressl?(3, 2, 2)
         begin
           sock = TCPSocket.new("127.0.0.1", port)
           ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
@@ -1501,12 +1509,13 @@ end
     end
   end
 
-  def test_ecdh_curves
+  def test_ecdh_curves_tls12
     pend "EC is disabled" unless defined?(OpenSSL::PKey::EC)
 
     ctx_proc = -> ctx {
       # Enable both ECDHE (~ TLS 1.2) cipher suites and TLS 1.3
-      ctx.ciphers = "DEFAULT:!kRSA:!kEDH"
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION
+      ctx.ciphers = "kEECDH"
       ctx.ecdh_curves = "P-384:P-521"
     }
     start_server(ctx_proc: ctx_proc, ignore_listener_error: true) do |port|
@@ -1515,13 +1524,9 @@ end
 
       server_connect(port, ctx) { |ssl|
         cs = ssl.cipher[0]
-        if /\ATLS/ =~ cs # Is TLS 1.3 is used?
+        assert_match (/\AECDH/), cs
+        if ssl.respond_to?(:tmp_key)
           assert_equal "secp384r1", ssl.tmp_key.group.curve_name
-        else
-          assert_match (/\AECDH/), cs
-          if ssl.respond_to?(:tmp_key)
-            assert_equal "secp384r1", ssl.tmp_key.group.curve_name
-          end
         end
         ssl.puts "abc"; assert_equal "abc\n", ssl.gets
       }
@@ -1545,6 +1550,26 @@ end
     end
   end
 
+  def test_ecdh_curves_tls13
+    pend "EC is disabled" unless defined?(OpenSSL::PKey::EC)
+    pend "TLS 1.3 not supported" unless tls13_supported?
+
+    ctx_proc = -> ctx {
+      # Assume TLS 1.3 is enabled and chosen by default
+      ctx.ecdh_curves = "P-384:P-521"
+    }
+    start_server(ctx_proc: ctx_proc, ignore_listener_error: true) do |port|
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.ecdh_curves = "P-256:P-384" # disable P-521
+
+      server_connect(port, ctx) { |ssl|
+        assert_equal "TLSv1.3", ssl.ssl_version
+        assert_equal "secp384r1", ssl.tmp_key.group.curve_name
+        ssl.puts "abc"; assert_equal "abc\n", ssl.gets
+      }
+    end
+  end
+
   def test_security_level
     ctx = OpenSSL::SSL::SSLContext.new
     begin
diff --git a/test/test_ssl_session.rb b/test/test_ssl_session.rb
index e199f86d..1d82aebf 100644
--- a/test/test_ssl_session.rb
+++ b/test/test_ssl_session.rb
@@ -122,6 +122,7 @@ __EOS__
       ctx.options &= ~OpenSSL::SSL::OP_NO_TICKET
       # Disable server-side session cache which is enabled by default
       ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_OFF
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
     }
     start_server(ctx_proc: ctx_proc) do |port|
       sess1 = server_connect_with_session(port, nil, nil) { |ssl|
diff --git a/test/utils.rb b/test/utils.rb
index bf191630..34c89a2e 100644
--- a/test/utils.rb
+++ b/test/utils.rb
@@ -181,6 +181,14 @@ class OpenSSL::SSLTestCase < OpenSSL::TestCase
   rescue
   end
 
+  def tls13_supported?
+    return false unless defined?(OpenSSL::SSL::TLS1_3_VERSION)
+    ctx = OpenSSL::SSL::SSLContext.new
+    ctx.min_version = ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
+    true
+  rescue
+  end
+
   def readwrite_loop(ctx, ssl)
     while line = ssl.gets
       ssl.write(line)