From 74ef8c0cc56b840b772240f2ee2b0fc0aafa2743 Mon Sep 17 00:00:00 2001
From: Kazuki Yamaguchi <k@rhe.jp>
Date: Mon, 24 Feb 2020 21:12:22 +0900
Subject: ssl: set verify error code in the case of verify_hostname failure

When the verify_hostname option is enabled, the hostname verification is
done before calling verify_callback provided by the user.

The callback should be notified of the hostname verification failure.
OpenSSL::X509::StoreContext's error code must be set to an appropriate
value rather than OpenSSL::X509::V_OK.

If the constant X509_V_ERR_HOSTNAME_MISMATCH is available (OpenSSL >=
1.0.2), use it. Otherwise use the generic X509_V_ERR_CERT_REJECTED.

Reference: https://github.com/ruby/openssl/issues/244
Fixes: 028e495734e9 ("ssl: add verify_hostname option to SSLContext", 2016-06-27)
---
 ext/openssl/ossl_ssl.c |  9 ++++++++-
 test/test_ssl.rb       | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 5422e699..3d076633 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -350,7 +350,14 @@ ossl_ssl_verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
 	    rb_ivar_set(ssl_obj, ID_callback_state, INT2NUM(status));
 	    return 0;
 	}
-	preverify_ok = ret == Qtrue;
+        if (ret != Qtrue) {
+            preverify_ok = 0;
+#if defined(X509_V_ERR_HOSTNAME_MISMATCH)
+            X509_STORE_CTX_set_error(ctx, X509_V_ERR_HOSTNAME_MISMATCH);
+#else
+            X509_STORE_CTX_set_error(ctx, X509_V_ERR_CERT_REJECTED);
+#endif
+        }
     }
 
     return ossl_verify_cb_call(cb, preverify_ok, ctx);
diff --git a/test/test_ssl.rb b/test/test_ssl.rb
index 408c7d82..8263e58a 100644
--- a/test/test_ssl.rb
+++ b/test/test_ssl.rb
@@ -752,6 +752,46 @@ class OpenSSL::TestSSL < OpenSSL::SSLTestCase
     end
   end
 
+  def test_verify_hostname_failure_error_code
+    ctx_proc = proc { |ctx|
+      exts = [
+        ["keyUsage", "keyEncipherment,digitalSignature", true],
+        ["subjectAltName", "DNS:a.example.com"],
+      ]
+      ctx.cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key)
+      ctx.key = @svr_key
+    }
+
+    start_server(ctx_proc: ctx_proc, ignore_listener_error: true) do |port|
+      verify_callback_ok = verify_callback_err = nil
+
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_hostname = true
+      ctx.cert_store = OpenSSL::X509::Store.new
+      ctx.cert_store.add_cert(@ca_cert)
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      ctx.verify_callback = -> (preverify_ok, store_ctx) {
+        verify_callback_ok = preverify_ok
+        verify_callback_err = store_ctx.error
+        preverify_ok
+      }
+
+      begin
+        sock = TCPSocket.new("127.0.0.1", port)
+        ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
+        ssl.hostname = "b.example.com"
+        assert_handshake_error { ssl.connect }
+        assert_equal false, verify_callback_ok
+        code_expected = openssl?(1, 0, 2) || defined?(OpenSSL::X509::V_ERR_HOSTNAME_MISMATCH) ?
+          OpenSSL::X509::V_ERR_HOSTNAME_MISMATCH :
+          OpenSSL::X509::V_ERR_CERT_REJECTED
+        assert_equal code_expected, verify_callback_err
+      ensure
+        sock&.close
+      end
+    end
+  end
+
   def test_unset_OP_ALL
     ctx_proc = Proc.new { |ctx|
       # If OP_DONT_INSERT_EMPTY_FRAGMENTS is not defined, this test is
-- 
cgit v1.2.3