ssl: set verify error code in the case of verify_hostname failureky/ssl-fix-verify-hostname-set-error-code

When the verify_hostname option is enabled, the hostname verification is
done before calling verify_callback provided by the user.

The callback should be notified of the hostname verification failure.
OpenSSL::X509::StoreContext's error code must be set to an appropriate
value rather than OpenSSL::X509::V_OK.

If the constant X509_V_ERR_HOSTNAME_MISMATCH is available (OpenSSL >=
1.0.2), use it. Otherwise use the generic X509_V_ERR_CERT_REJECTED.

Reference: https://github.com/ruby/openssl/issues/244
Fixes: 028e495734e9 ("ssl: add verify_hostname option to SSLContext", 2016-06-27)

1 files changed, 8 insertions, 1 deletions

diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 5422e699..3d076633 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -350,7 +350,14 @@ ossl_ssl_verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
 	    rb_ivar_set(ssl_obj, ID_callback_state, INT2NUM(status));
 	    return 0;
 	}
-	preverify_ok = ret == Qtrue;
+        if (ret != Qtrue) {
+            preverify_ok = 0;
+#if defined(X509_V_ERR_HOSTNAME_MISMATCH)
+            X509_STORE_CTX_set_error(ctx, X509_V_ERR_HOSTNAME_MISMATCH);
+#else
+            X509_STORE_CTX_set_error(ctx, X509_V_ERR_CERT_REJECTED);
+#endif
+        }
     }
 
     return ossl_verify_cb_call(cb, preverify_ok, ctx);