openssl: clear OpenSSL error queue before return to Ruby

* ext/openssl/ossl_x509cert.c (ossl_x509_verify): X509_verify()
  family may put errors on 0 return (0 means verification failure).
  Clear OpenSSL error queue before return to Ruby. Since the queue is
  thread global, remaining errors in the queue can cause an unexpected
  error in the next OpenSSL operation.  [ruby-core:48284] [Bug #7215]

* ext/openssl/ossl_x509crl.c (ossl_x509crl_verify): ditto.

* ext/openssl/ossl_x509req.c (ossl_x509req_verify): ditto.

* ext/openssl/ossl_x509store.c (ossl_x509stctx_verify): ditto.

* ext/openssl/ossl_pkey_dh.c (dh_generate): clear the OpenSSL error
  queue before re-raising exception.

* ext/openssl/ossl_pkey_dsa.c (dsa_generate): ditto.

* ext/openssl/ossl_pkey_rsa.c (rsa_generate): ditto.

* ext/openssl/ossl_ssl.c (ossl_start_ssl): ditto.

* test/openssl: check that OpenSSL.errors is empty every time after
  running a test case.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55051 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

8 files changed, 56 insertions, 31 deletions

diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index 2f79bfb..19c517f 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -129,7 +129,11 @@ dh_generate(int size, int gen)
 
     if (!gen_arg.result) {
 	DH_free(dh);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* Clear OpenSSL error queue before re-raising. */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 #else
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 2e42a0c..4c0c3f1 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -135,7 +135,14 @@ dsa_generate(int size)
     }
     if (!gen_arg.result) {
 	DSA_free(dsa);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* Clear OpenSSL error queue before re-raising. By the way, the
+	     * documentation of DSA_generate_parameters_ex() says the error code
+	     * can be obtained by ERR_get_error(), but the default
+	     * implementation, dsa_builtin_paramgen() doesn't put any error... */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 #else
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 20b993a..6ad9f3e 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -139,7 +139,11 @@ rsa_generate(int size, unsigned long exp)
     if (!gen_arg.result) {
 	BN_free(e);
 	RSA_free(rsa);
-	if (cb_arg.state) rb_jump_tag(cb_arg.state);
+	if (cb_arg.state) {
+	    /* must clear OpenSSL error stack */
+	    ossl_clear_error();
+	    rb_jump_tag(cb_arg.state);
+	}
 	return 0;
     }
 
diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 1079710..938e36f 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -1288,8 +1288,11 @@ ossl_start_ssl(VALUE self, int (*func)(), const char *funcname, VALUE opts)
 	ret = func(ssl);
 
         cb_state = rb_ivar_get(self, ID_callback_state);
-        if (!NIL_P(cb_state))
-            rb_jump_tag(NUM2INT(cb_state));
+        if (!NIL_P(cb_state)) {
+	    /* must cleanup OpenSSL error stack before re-raising */
+	    ossl_clear_error();
+	    rb_jump_tag(NUM2INT(cb_state));
+	}
 
 	if (ret > 0)
 	    break;
diff --git a/ext/openssl/ossl_x509cert.c b/ext/openssl/ossl_x509cert.c
index 4dafae1..226704e 100644
--- a/ext/openssl/ossl_x509cert.c
+++ b/ext/openssl/ossl_x509cert.c
@@ -591,18 +591,19 @@ ossl_x509_verify(VALUE self, VALUE key)
 {
     X509 *x509;
     EVP_PKEY *pkey;
-    int i;
 
     pkey = GetPKeyPtr(key); /* NO NEED TO DUP */
     GetX509(self, x509);
-    if ((i = X509_verify(x509, pkey)) < 0) {
-	ossl_raise(eX509CertError, NULL);
-    }
-    if (i > 0) {
+
+    switch (X509_verify(x509, pkey)) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CertError, NULL);
     }
-
-    return Qfalse;
 }
 
 /*
diff --git a/ext/openssl/ossl_x509crl.c b/ext/openssl/ossl_x509crl.c
index f64712e..a660ccc 100644
--- a/ext/openssl/ossl_x509crl.c
+++ b/ext/openssl/ossl_x509crl.c
@@ -360,17 +360,17 @@ static VALUE
 ossl_x509crl_verify(VALUE self, VALUE key)
 {
     X509_CRL *crl;
-    int ret;
 
     GetX509CRL(self, crl);
-    if ((ret = X509_CRL_verify(crl, GetPKeyPtr(key))) < 0) {
-	ossl_raise(eX509CRLError, NULL);
-    }
-    if (ret == 1) {
+    switch (X509_CRL_verify(crl, GetPKeyPtr(key))) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CRLError, NULL);
     }
-
-    return Qfalse;
 }
 
 static VALUE
diff --git a/ext/openssl/ossl_x509req.c b/ext/openssl/ossl_x509req.c
index e5ce088..c1cdca5 100644
--- a/ext/openssl/ossl_x509req.c
+++ b/ext/openssl/ossl_x509req.c
@@ -375,18 +375,18 @@ ossl_x509req_verify(VALUE self, VALUE key)
 {
     X509_REQ *req;
     EVP_PKEY *pkey;
-    int i;
 
     GetX509Req(self, req);
     pkey = GetPKeyPtr(key); /* NO NEED TO DUP */
-    if ((i = X509_REQ_verify(req, pkey)) < 0) {
-	ossl_raise(eX509ReqError, NULL);
-    }
-    if (i > 0) {
+    switch (X509_REQ_verify(req, pkey)) {
+      case 1:
 	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509ReqError, NULL);
     }
-
-    return Qfalse;
 }
 
 static VALUE
diff --git a/ext/openssl/ossl_x509store.c b/ext/openssl/ossl_x509store.c
index bb6fe14..aca25b1 100644
--- a/ext/openssl/ossl_x509store.c
+++ b/ext/openssl/ossl_x509store.c
@@ -464,14 +464,20 @@ static VALUE
 ossl_x509stctx_verify(VALUE self)
 {
     X509_STORE_CTX *ctx;
-    int result;
 
     GetX509StCtx(self, ctx);
     X509_STORE_CTX_set_ex_data(ctx, ossl_verify_cb_idx,
-                               (void*)rb_iv_get(self, "@verify_callback"));
-    result = X509_verify_cert(ctx);
-
-    return result ? Qtrue : Qfalse;
+			       (void *)rb_iv_get(self, "@verify_callback"));
+
+    switch (X509_verify_cert(ctx)) {
+      case 1:
+	return Qtrue;
+      case 0:
+	ossl_clear_error();
+	return Qfalse;
+      default:
+	ossl_raise(eX509CertError, NULL);
+    }
 }
 
 static VALUE