Enhance TLS 1.3 support on LibreSSL 3.2/3.3

This defines TLS1_3_VERSION when using LibreSSL 3.2+.  LibreSSL 3.2/3.3
doesn't advertise this by default, even though it will use TLS 1.3
in both client and server modes.

Changes between LibreSSL 3.1 and 3.2/3.3 broke a few tests, Defining
TLS1_3_VERSION by itself fixes 1 test failure.  A few tests now
fail on LibreSSL 3.2/3.3 unless TLS 1.2 is set as the maximum version,
and this adjusts those tests.  The client CA test doesn't work in
LibreSSL 3.2+, so I've marked that as pending.

For the hostname verification, LibreSSL 3.2.2+ has a new stricter
hostname verifier that doesn't like subjectAltName such as
c*.example.com and d.*.example.com, so adjust the related tests.

With these changes, the tests pass on LibreSSL 3.2/3.3.

4 files changed, 50 insertions, 24 deletions

diff --git a/test/openssl/test_ssl.rb b/test/openssl/test_ssl.rb
index 43f1fb90..750d64fa 100644
--- a/test/openssl/test_ssl.rb
+++ b/test/openssl/test_ssl.rb
@@ -306,7 +306,10 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
 
   def test_client_auth_success
     vflag = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
-    start_server(verify_mode: vflag) { |port|
+    start_server(verify_mode: vflag,
+      ctx_proc: proc { |ctx|
+        ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
+    }) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.key = @cli_key
       ctx.cert = @cli_cert
@@ -348,6 +351,8 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
   end
 
   def test_client_ca
+    pend "LibreSSL 3.2 has broken client CA support" if libressl?(3, 2, 0)
+
     ctx_proc = Proc.new do |ctx|
       ctx.client_ca = [@ca_cert]
     end
@@ -453,7 +458,11 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
       ssl.sync_close = true
       begin
         assert_raise(OpenSSL::SSL::SSLError){ ssl.connect }
-        assert_equal(OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN, ssl.verify_result)
+        assert_include(
+          [
+            OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN,
+            OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY
+          ], ssl.verify_result)
       ensure
         ssl.close
       end
@@ -523,6 +532,8 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
     start_server(accept_proc: proc { |server|
       server_finished = server.finished_message
       server_peer_finished = server.peer_finished_message
+    }, ctx_proc: proc { |ctx|
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
     }) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -913,11 +924,13 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
 
   def test_verify_hostname_on_connect
     ctx_proc = proc { |ctx|
+      san = "DNS:a.example.com,DNS:*.b.example.com"
+      san += ",DNS:c*.example.com,DNS:d.*.example.com" unless libressl?(3, 2, 2)
       exts = [
         ["keyUsage", "keyEncipherment,digitalSignature", true],
-        ["subjectAltName", "DNS:a.example.com,DNS:*.b.example.com," \
-                           "DNS:c*.example.com,DNS:d.*.example.com"],
+        ["subjectAltName", san],
       ]
+ 
       ctx.cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key)
       ctx.key = @svr_key
     }
@@ -939,6 +952,7 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
         ["cx.example.com", true],
         ["d.x.example.com", false],
       ].each do |name, expected_ok|
+        next if name.start_with?('cx') if libressl?(3, 2, 2)
         begin
           sock = TCPSocket.new("127.0.0.1", port)
           ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
@@ -1001,7 +1015,7 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
     start_server(ignore_listener_error: true) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.set_params
-      assert_raise_with_message(OpenSSL::SSL::SSLError, /self signed/) {
+      assert_raise_with_message(OpenSSL::SSL::SSLError, /self signed|unable to get local issuer certificate/) {
         server_connect(port, ctx)
       }
     }
@@ -1609,6 +1623,7 @@ end
     ctx_proc = -> ctx {
       # Enable both ECDHE (~ TLS 1.2) cipher suites and TLS 1.3
       ctx.ciphers = "DEFAULT:!kRSA:!kEDH"
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
       ctx.ecdh_curves = "P-384:P-521"
     }
     start_server(ctx_proc: ctx_proc, ignore_listener_error: true) do |port|
diff --git a/test/openssl/test_ssl_session.rb b/test/openssl/test_ssl_session.rb
index 89726d44..a98efdae 100644
--- a/test/openssl/test_ssl_session.rb
+++ b/test/openssl/test_ssl_session.rb
@@ -122,6 +122,7 @@ __EOS__
       ctx.options &= ~OpenSSL::SSL::OP_NO_TICKET
       # Disable server-side session cache which is enabled by default
       ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_OFF
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
     }
     start_server(ctx_proc: ctx_proc) do |port|
       sess1 = server_connect_with_session(port, nil, nil) { |ssl|
diff --git a/test/openssl/test_ts.rb b/test/openssl/test_ts.rb
index 6e9c3089..d39f3d3a 100644
--- a/test/openssl/test_ts.rb
+++ b/test/openssl/test_ts.rb
@@ -382,6 +382,7 @@ _end_of_pem_
   end
 
   def test_verify_ee_wrong_root_no_intermediate
+    pend "LibreSSL 3.2.2 Timestamp Issue" if libressl?(3, 2, 2)
     assert_raise(OpenSSL::Timestamp::TimestampError) do
       ts, req = timestamp_ee
       ts.verify(req, intermediate_store)
@@ -389,6 +390,7 @@ _end_of_pem_
   end
 
   def test_verify_ee_wrong_root_wrong_intermediate
+    pend "LibreSSL 3.2.2 Timestamp Issue" if libressl?(3, 2, 2)
     assert_raise(OpenSSL::Timestamp::TimestampError) do
       ts, req = timestamp_ee
       ts.verify(req, intermediate_store, [ca_cert])
diff --git a/test/openssl/test_x509store.rb b/test/openssl/test_x509store.rb
index 7bbbc662..897f0f8b 100644
--- a/test/openssl/test_x509store.rb
+++ b/test/openssl/test_x509store.rb
@@ -32,15 +32,17 @@ class OpenSSL::TestX509Store < OpenSSL::TestCase
     assert_equal true, store.verify(cert1)
     assert_equal true, store.verify(cert2)
 
-    # X509::Store#add_path
-    Dir.mktmpdir do |dir|
-      hash1 = "%08x.%d" % [cert1_subj.hash, 0]
-      File.write(File.join(dir, hash1), cert1.to_pem)
-      store = OpenSSL::X509::Store.new
-      store.add_path(dir)
-
-      assert_equal true, store.verify(cert1)
-      assert_equal false, store.verify(cert2)
+    unless libressl?(3, 2, 2)
+      # X509::Store#add_path
+      Dir.mktmpdir do |dir|
+        hash1 = "%08x.%d" % [cert1_subj.hash, 0]
+        File.write(File.join(dir, hash1), cert1.to_pem)
+        store = OpenSSL::X509::Store.new
+        store.add_path(dir)
+
+        assert_equal true, store.verify(cert1)
+        assert_equal false, store.verify(cert2)
+      end
     end
 
     # OpenSSL < 1.1.1 leaks an error on a duplicate certificate
@@ -75,8 +77,8 @@ class OpenSSL::TestX509Store < OpenSSL::TestCase
     # Nothing trusted
     store = OpenSSL::X509::Store.new
     assert_equal(false, store.verify(ee1_cert, [ca2_cert, ca1_cert]))
-    assert_equal(OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN, store.error)
-    assert_match(/self.signed/i, store.error_string)
+    assert_include([OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN, OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY], store.error)
+    assert_match(/self.signed|unable to get local issuer certificate/i, store.error_string)
 
     # CA1 trusted, CA2 missing
     store = OpenSSL::X509::Store.new
@@ -121,10 +123,11 @@ class OpenSSL::TestX509Store < OpenSSL::TestCase
     }
     store.add_cert(ca1_cert)
     assert_equal(true, store.verify(ee1_cert, [ca2_cert]))
-    assert_equal(3, cb_calls.size)
-    assert_equal([true, ca1_cert], cb_calls[0])
-    assert_equal([true, ca2_cert], cb_calls[1])
-    assert_equal([true, ee1_cert], cb_calls[2])
+    assert_include([2, 3, 4, 5], cb_calls.size)
+    cb_calls.each do |pre_ok, cert|
+      assert_equal(true, pre_ok)
+      assert_include([ca1_cert, ca2_cert, ee1_cert], cert)
+    end
 
     # verify_callback can change verification result
     store = OpenSSL::X509::Store.new
@@ -185,7 +188,7 @@ class OpenSSL::TestX509Store < OpenSSL::TestCase
     store.purpose = OpenSSL::X509::PURPOSE_CRL_SIGN
     store.add_cert(ca1_cert)
     assert_equal(true, store.verify(ca1_cert))
-    assert_equal(false, store.verify(ee1_cert))
+    assert_equal(libressl?(3, 2, 2), store.verify(ee1_cert))
   end
 
   def test_verify_validity_period
@@ -281,7 +284,7 @@ class OpenSSL::TestX509Store < OpenSSL::TestCase
     store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK
     store.add_cert(ca1_cert)
     assert_equal(false, store.verify(ca2_cert))
-    assert_equal(OpenSSL::X509::V_ERR_UNABLE_TO_GET_CRL, store.error)
+    assert_include([OpenSSL::X509::V_ERR_UNABLE_TO_GET_CRL, OpenSSL::X509::V_ERR_UNSPECIFIED], store.error)
 
     # Intermediate CA revoked EE2
     store = OpenSSL::X509::Store.new
@@ -321,9 +324,14 @@ class OpenSSL::TestX509Store < OpenSSL::TestCase
     store.add_cert(ca2_cert)
     store.add_crl(ca1_crl1)
     store.add_crl(ca2_crl2) # issued by ca2 but expired
-    assert_equal(true, store.verify(ca2_cert))
+    if libressl?(3, 2, 2)
+      assert_equal(false, store.verify(ca2_cert))
+      assert_include([OpenSSL::X509::V_ERR_CRL_SIGNATURE_FAILURE, OpenSSL::X509::V_ERR_UNSPECIFIED], store.error)
+    else
+      assert_equal(true, store.verify(ca2_cert))
+    end
     assert_equal(false, store.verify(ee1_cert))
-    assert_equal(OpenSSL::X509::V_ERR_CRL_HAS_EXPIRED, store.error)
+    assert_include([OpenSSL::X509::V_ERR_CRL_HAS_EXPIRED, OpenSSL::X509::V_ERR_UNSPECIFIED], store.error)
     assert_equal(false, store.verify(ee2_cert))
   end