pkey: resume key generation after interruptky/pkey-generate-interrupt-resume

Key/parameter generation (OpenSSL::PKey::*.{new,generate}) immediately
aborts when it is done with GVL released (in other words, no block is
given) and the thread is interrupted (e.g., by a signal) during the
operation.

Have ossl_generate_cb_2() acquire GVL and call rb_thread_check_ints()
if needed to process the pending interrupt rather than abort the
operation completely by returning 0.

Reference: https://bugs.ruby-lang.org/issues/14882

2 files changed, 26 insertions, 4 deletions

diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index aad3e2e4..962964ee 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -20,6 +20,21 @@ static ID id_private_q;
 /*
  * callback for generating keys
  */
+static VALUE
+call_check_ints0(VALUE arg)
+{
+    rb_thread_check_ints();
+    return Qnil;
+}
+
+static void *
+call_check_ints(void *arg)
+{
+    int state;
+    rb_protect(call_check_ints0, Qnil, &state);
+    return (void *)(VALUE)state;
+}
+
 int
 ossl_generate_cb_2(int p, int n, BN_GENCB *cb)
 {
@@ -38,11 +53,18 @@ ossl_generate_cb_2(int p, int n, BN_GENCB *cb)
 	*/
 	rb_protect(rb_yield, ary, &state);
 	if (state) {
-	    arg->stop = 1;
 	    arg->state = state;
+	    return 0;
+	}
+    }
+    if (arg->interrupted) {
+	arg->interrupted = 0;
+	state = (int)(VALUE)rb_thread_call_with_gvl(call_check_ints, NULL);
+	if (state) {
+	    arg->state = state;
+	    return 0;
 	}
     }
-    if (arg->stop) return 0;
     return 1;
 }
 
@@ -50,7 +72,7 @@ void
 ossl_generate_cb_stop(void *ptr)
 {
     struct ossl_generate_cb_arg *arg = (struct ossl_generate_cb_arg *)ptr;
-    arg->stop = 1;
+    arg->interrupted = 1;
 }
 
 static void
diff --git a/ext/openssl/ossl_pkey.h b/ext/openssl/ossl_pkey.h
index a0b49744..5c8ccc0f 100644
--- a/ext/openssl/ossl_pkey.h
+++ b/ext/openssl/ossl_pkey.h
@@ -41,7 +41,7 @@ extern const rb_data_type_t ossl_evp_pkey_type;
 
 struct ossl_generate_cb_arg {
     int yield;
-    int stop;
+    int interrupted;
     int state;
 };
 int ossl_generate_cb_2(int p, int n, BN_GENCB *cb);