test/test_ssl_session: skip tests for session_remove_cbky/skip-session-remove-cb-tests

In OpenSSL < 1.1.0, the session_remove_cb callback is called inside the
global lock for CRYPTO_LOCK_SSL_CTX which is shared across the entire
process, not just for the specific SSL_CTX object. It is possible that
the callback releases GVL while the lock for CRYPTO_LOCK_SSL_CTX is
held, causing another thread calling an OpenSSL function that tries to
acquire the same lock stuck forever.

Add a note about the possible deadlock to the docs for
SSLContext#session_remove_cb=, and skip the relevant test cases unless
the OSSL_TEST_ALL environment variable is set to 1.

A deadlock due to this issue is observed:

  http://ci.rvm.jp/results/trunk-test@frontier/104428

2 files changed, 44 insertions, 18 deletions

diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index aa2dbbc8..736f4924 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -2446,6 +2446,10 @@ Init_ossl_ssl(void)
      * A callback invoked when a session is removed from the internal cache.
      *
      * The callback is invoked with an SSLContext and a Session.
+     *
+     * IMPORTANT NOTE: It is currently not possible to use this safely in a
+     * multi-threaded application. The callback is called inside a global lock
+     * and it can randomly cause deadlock on Ruby thread switching.
      */
     rb_attr(cSSLContext, rb_intern("session_remove_cb"), 1, 1, Qfalse);
 
diff --git a/test/test_ssl_session.rb b/test/test_ssl_session.rb
index 9d0e5a2d..af8c65b1 100644
--- a/test/test_ssl_session.rb
+++ b/test/test_ssl_session.rb
@@ -215,6 +215,10 @@ __EOS__
     end
   end
 
+  # Skipping tests that use session_remove_cb by default because it may cause
+  # deadlock.
+  TEST_SESSION_REMOVE_CB = ENV["OSSL_TEST_ALL"] == "1"
+
   def test_ctx_client_session_cb
     pend "TLS 1.2 is not supported" unless tls12_supported?
 
@@ -227,11 +231,13 @@ __EOS__
         sock, sess = ary
         called[:new] = [sock, sess]
       }
-      ctx.session_remove_cb = lambda { |ary|
-        ctx, sess = ary
-        called[:remove] = [ctx, sess]
-        # any resulting value is OK (ignored)
-      }
+      if TEST_SESSION_REMOVE_CB
+        ctx.session_remove_cb = lambda { |ary|
+          ctx, sess = ary
+          called[:remove] = [ctx, sess]
+          # any resulting value is OK (ignored)
+        }
+      end
 
       server_connect_with_session(port, ctx, nil) { |ssl|
         assert_equal(1, ctx.session_cache_stats[:cache_num])
@@ -239,7 +245,9 @@ __EOS__
         assert_equal([ssl, ssl.session], called[:new])
         assert(ctx.session_remove(ssl.session))
         assert(!ctx.session_remove(ssl.session))
-        assert_equal([ctx, ssl.session], called[:remove])
+        if TEST_SESSION_REMOVE_CB
+          assert_equal([ctx, ssl.session], called[:remove])
+        end
       }
     end
   end
@@ -275,10 +283,12 @@ __EOS__
         last_server_session = sess
       }
 
-      ctx.session_remove_cb = lambda { |ary|
-        _ctx, sess = ary
-        called[:remove] = sess
-      }
+      if TEST_SESSION_REMOVE_CB
+        ctx.session_remove_cb = lambda { |ary|
+          _ctx, sess = ary
+          called[:remove] = sess
+        }
+      end
     }
     start_server(ctx_proc: ctx_proc) do |port|
       connections = 0
@@ -290,7 +300,9 @@ __EOS__
       assert_nil called[:get]
       assert_not_nil called[:new]
       assert_equal sess0.id, called[:new].id
-      assert_nil called[:remove]
+      if TEST_SESSION_REMOVE_CB
+        assert_nil called[:remove]
+      end
       called.clear
 
       # Internal cache hit
@@ -302,12 +314,16 @@ __EOS__
       }
       assert_nil called[:get]
       assert_nil called[:new]
-      assert_nil called[:remove]
+      if TEST_SESSION_REMOVE_CB
+        assert_nil called[:remove]
+      end
       called.clear
 
       sctx.flush_sessions(Time.now + 10000)
-      assert_not_nil called[:remove]
-      assert_equal sess0.id, called[:remove].id
+      if TEST_SESSION_REMOVE_CB
+        assert_not_nil called[:remove]
+        assert_equal sess0.id, called[:remove].id
+      end
       called.clear
 
       # External cache hit
@@ -325,12 +341,16 @@ __EOS__
       assert_equal sess0.id, sess2.id
       assert_equal sess0.id, called[:get]
       assert_nil called[:new]
-      assert_nil called[:remove]
+      if TEST_SESSION_REMOVE_CB
+        assert_nil called[:remove]
+      end
       called.clear
 
       sctx.flush_sessions(Time.now + 10000)
-      assert_not_nil called[:remove]
-      assert_equal sess0.id, called[:remove].id
+      if TEST_SESSION_REMOVE_CB
+        assert_not_nil called[:remove]
+        assert_equal sess0.id, called[:remove].id
+      end
       called.clear
 
       # Cache miss
@@ -344,7 +364,9 @@ __EOS__
       assert_equal sess0.id, called[:get]
       assert_not_nil called[:new]
       assert_equal sess3.id, called[:new].id
-      assert_nil called[:remove]
+      if TEST_SESSION_REMOVE_CB
+        assert_nil called[:remove]
+      end
     end
   end