openssl: clear OpenSSL error queue before return to Ruby

* ext/openssl/ossl_x509cert.c (ossl_x509_verify): X509_verify()
  family may put errors on 0 return (0 means verification failure).
  Clear OpenSSL error queue before return to Ruby. Since the queue is
  thread global, remaining errors in the queue can cause an unexpected
  error in the next OpenSSL operation.  [ruby-core:48284] [Bug #7215]

* ext/openssl/ossl_x509crl.c (ossl_x509crl_verify): ditto.

* ext/openssl/ossl_x509req.c (ossl_x509req_verify): ditto.

* ext/openssl/ossl_x509store.c (ossl_x509stctx_verify): ditto.

* ext/openssl/ossl_pkey_dh.c (dh_generate): clear the OpenSSL error
  queue before re-raising exception.

* ext/openssl/ossl_pkey_dsa.c (dsa_generate): ditto.

* ext/openssl/ossl_pkey_rsa.c (rsa_generate): ditto.

* ext/openssl/ossl_ssl.c (ossl_start_ssl): ditto.

* test/openssl: check that OpenSSL.errors is empty every time after
  running a test case.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55051 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

8 files changed, 56 insertions, 31 deletions

diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index 2f79bfb2..19c517fd 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -129,7 +129,11 @@ dh_generate(int size, int gen)
 
     if (!gen_arg.result) {
 	DH_free(dh);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* Clear OpenSSL error queue before re-raising. */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 #else
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 2e42a0ce..4c0c3f1b 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -135,7 +135,14 @@ dsa_generate(int size)
     }
     if (!gen_arg.result) {
 	DSA_free(dsa);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* Clear OpenSSL error queue before re-raising. By the way, the
+	     * documentation of DSA_generate_parameters_ex() says the error code
+	     * can be obtained by ERR_get_error(), but the default
+	     * implementation, dsa_builtin_paramgen() doesn't put any error... */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 #else
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 20b993ab..6ad9f3ed 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -139,7 +139,11 @@ rsa_generate(int size, unsigned long exp)
     if (!gen_arg.result) {
 	BN_free(e);
 	RSA_free(rsa);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* must clear OpenSSL error stack */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 
diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 10797109..938e36f1 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -1288,8 +1288,11 @@ ossl_start_ssl(VALUE self, int (*func)(), const char *funcname, VALUE opts)
 	ret = func(ssl);
 
         cb_state = rb_ivar_get(self, ID_callback_state);
-        if (!NIL_P(cb_state))
-            rb_jump_tag(NUM2INT(cb_state));
+        if (!NIL_P(cb_state)) {
+	    /* must cleanup OpenSSL error stack before re-raising */
+	    ossl_clear_error();
+	    rb_jump_tag(NUM2INT(cb_state));
+	}
 
 	if (ret > 0)
 	    break;
diff --git a/ext/openssl/ossl_x509cert.c b/ext/openssl/ossl_x509cert.c
index 4dafae17..226704ef 100644
--- a/ext/openssl/ossl_x509cert.c
+++ b/ext/openssl/ossl_x509cert.c
@@ -591,18 +591,19 @@ ossl_x509_verify(VALUE self, VALUE key)
 {
     X509 *x509;
     EVP_PKEY *pkey;
-    int i;
 
     pkey = GetPKeyPtr(key); /* NO NEED TO DUP */
     GetX509(self, x509);
-    if ((i = X509_verify(x509, pkey)) < 0) {
-	ossl_raise(eX509CertError, NULL);
-    }
-    if (i > 0) {
+
+    switch (X509_verify(x509, pkey)) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CertError, NULL);
     }
-
-    return Qfalse;
 }
 
 /*
diff --git a/ext/openssl/ossl_x509crl.c b/ext/openssl/ossl_x509crl.c
index f64712ef..a660ccce 100644
--- a/ext/openssl/ossl_x509crl.c
+++ b/ext/openssl/ossl_x509crl.c
@@ -360,17 +360,17 @@ static VALUE
 ossl_x509crl_verify(VALUE self, VALUE key)
 {
     X509_CRL *crl;
-    int ret;
 
     GetX509CRL(self, crl);
-    if ((ret = X509_CRL_verify(crl, GetPKeyPtr(key))) < 0) {
-	ossl_raise(eX509CRLError, NULL);
-    }
-    if (ret == 1) {
+    switch (X509_CRL_verify(crl, GetPKeyPtr(key))) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CRLError, NULL);
     }
-
-    return Qfalse;
 }
 
 static VALUE
diff --git a/ext/openssl/ossl_x509req.c b/ext/openssl/ossl_x509req.c
index e5ce088a..c1cdca5f 100644
--- a/ext/openssl/ossl_x509req.c
+++ b/ext/openssl/ossl_x509req.c
@@ -375,18 +375,18 @@ ossl_x509req_verify(VALUE self, VALUE key)
 {
     X509_REQ *req;
     EVP_PKEY *pkey;
-    int i;
 
     GetX509Req(self, req);
     pkey = GetPKeyPtr(key); /* NO NEED TO DUP */
-    if ((i = X509_REQ_verify(req, pkey)) < 0) {
-	ossl_raise(eX509ReqError, NULL);
-    }
-    if (i > 0) {
+    switch (X509_REQ_verify(req, pkey)) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509ReqError, NULL);
     }
-
-    return Qfalse;
 }
 
 static VALUE
diff --git a/ext/openssl/ossl_x509store.c b/ext/openssl/ossl_x509store.c
index bb6fe14d..aca25b15 100644
--- a/ext/openssl/ossl_x509store.c
+++ b/ext/openssl/ossl_x509store.c
@@ -464,14 +464,20 @@ static VALUE
 ossl_x509stctx_verify(VALUE self)
 {
     X509_STORE_CTX *ctx;
-    int result;
 
     GetX509StCtx(self, ctx);
     X509_STORE_CTX_set_ex_data(ctx, ossl_verify_cb_idx,
-                               (void*)rb_iv_get(self, "@verify_callback"));
-    result = X509_verify_cert(ctx);
-
-    return result ? Qtrue : Qfalse;
+			       (void *)rb_iv_get(self, "@verify_callback"));
+
+    switch (X509_verify_cert(ctx)) {
+      case 1:
+	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CertError, NULL);
+    }
 }
 
 static VALUE