ossl.c: make legacy locking callbacks reentrantky/fix-legacy-locking-callback-relock

Although it's not documented explicitly that the locking callbacks must
provide reentrant mutexes, it seems to be required.

Specifically, the session_remove_cb callback function of an SSL_CTX is
called in a critical section for CRYPTO_LOCK_SSL_CTX, which is shared
across the library. This leads, if the callback function calls another
OpenSSL function that will attempt to lock CRYPTO_LOCK_SSL_CTX, to
deadlock. SSL_CTX_free() is one example of such a function.

	http://ci.rvm.jp/results/trunk@P895/64001

1 files changed, 13 insertions, 1 deletions

diff --git a/ext/openssl/ossl.c b/ext/openssl/ossl.c
index 7d3ed6c4..88bb8f21 100644
--- a/ext/openssl/ossl.c
+++ b/ext/openssl/ossl.c
@@ -486,21 +486,33 @@ print_mem_leaks(VALUE self)
  */
 struct CRYPTO_dynlock_value {
     rb_nativethread_lock_t lock;
+    rb_nativethread_id_t owner;
+    size_t count;
 };
 
 static void
 ossl_lock_init(struct CRYPTO_dynlock_value *l)
 {
     rb_nativethread_lock_initialize(&l->lock);
+    l->count = 0;
 }
 
 static void
 ossl_lock_unlock(int mode, struct CRYPTO_dynlock_value *l)
 {
     if (mode & CRYPTO_LOCK) {
+	/* TODO: rb_nativethread_id_t is not necessarily compared with ==. */
+	rb_nativethread_id_t tid = rb_nativethread_self();
+	if (l->count && l->owner == tid) {
+	    l->count++;
+	    return;
+	}
 	rb_nativethread_lock_lock(&l->lock);
+	l->owner = tid;
+	l->count = 1;
     } else {
-	rb_nativethread_lock_unlock(&l->lock);
+	if (!--l->count)
+	    rb_nativethread_lock_unlock(&l->lock);
     }
 }